Browse Source

Move get_index and index cache into Library. Code is running.

Lev Israel 2 years ago
parent
commit
90bc1b60c6

+ 2 - 4
doc/index.rst

@@ -144,10 +144,8 @@ Version
 .. autoclass:: VersionSet
 
 
-get_index, Index, and CommentaryIndex 
--------------------------------------
-
-.. autofunction:: get_index
+Index, and CommentaryIndex
+--------------------------
 
 .. autoclass:: Index
 

+ 1 - 1
reader/templatetags/sefaria_tags.py

@@ -209,7 +209,7 @@ def lang_code(code):
 def text_category(text):
 	"""Returns the top level category for text"""
 	try:
-		i = m.get_index(text)
+		i = m.library.get_index(text)
 		result = mark_safe(getattr(i, "categories", ["[no cats]"])[0])
 	except:
 		result = "[text not found]"

+ 7 - 7
reader/views.py

@@ -272,7 +272,7 @@ def edit_text(request, ref=None, lang=None, version=None):
                 mode = text["mode"].capitalize()
                 initJSON = json.dumps(text)
         except:
-            index = get_index(ref)
+            index = library.get_index(ref)
             if index: # a commentator titlein
                 ref = None
                 initJSON = json.dumps({"mode": "add new", "newTitle": index.contents()['title']})
@@ -297,7 +297,7 @@ def edit_text_info(request, title=None, new_title=None):
     if title:
         # Edit Existing
         title = title.replace("_", " ")
-        i = get_index(title)
+        i = library.get_index(title)
         if not (request.user.is_staff or user_started_text(request.user.id, title)):
             return render_to_response('static/generic.html', {"title": "Permission Denied", "content": "The Text Info for %s is locked.<br><br>Please email hello@sefaria.org if you believe edits are needed." % title}, RequestContext(request))
         indexJSON = json.dumps(i.contents(v2=True) if "toc" in request.GET else i.contents())
@@ -308,7 +308,7 @@ def edit_text_info(request, title=None, new_title=None):
         # Add New
         new_title = new_title.replace("_", " ")
         try: # Redirect to edit path if this title already exists
-            i = get_index(new_title)
+            i = library.get_index(new_title)
             return redirect("/edit/textinfo/%s" % new_title)
         except:
             pass
@@ -581,7 +581,7 @@ def text_toc(request, oref):
     cat_slices    = [categories[:n+1] for n in range(len(categories))] # successive sublists of cats, for category links
 
     c_titles      = model.library.get_commentary_version_titles_on_book(title, with_commentary2=True)
-    c_indexes     = [get_index(commentary) for commentary in c_titles]
+    c_indexes     = [library.get_index(commentary) for commentary in c_titles]
     commentaries  = [i.toc_contents() for i in c_indexes]
 
     if index.is_complex():
@@ -880,7 +880,7 @@ def index_api(request, title, v2=False, raw=False):
     """
     if request.method == "GET":
         try:
-            i = model.get_index(title).contents(v2=v2, raw=raw)
+            i = library.get_index(title).contents(v2=v2, raw=raw)
         except InputError as e:
             node = library.get_schema_node(title)  # If the request were for v1 and fails, this falls back to v2.
             if not node:
@@ -909,7 +909,7 @@ def index_api(request, title, v2=False, raw=False):
         else:
             title = j.get("oldTitle", j.get("title"))
             try:
-                i = get_index(title) # Only allow staff and the person who submitted a text to edit
+                i = library.get_index(title) # Only allow staff and the person who submitted a text to edit
                 if not request.user.is_staff and not user_started_text(request.user.id, title):
                    return jsonResponse({"error": "{} is protected from change.<br/><br/>See a mistake?<br/>Email hello@sefaria.org.".format(title)})
             except BookNameError:
@@ -927,7 +927,7 @@ def index_api(request, title, v2=False, raw=False):
 
         title = title.replace("_", " ")
 
-        i = get_index(title)
+        i = library.get_index(title)
 
         i.delete()
         record_index_deletion(title, request.user.id)

+ 1 - 1
sefaria/clean.py

@@ -83,7 +83,7 @@ def remove_old_counts():
     for count in counts:
         if count.get("title", None):
             try:
-                model.get_index(count["title"])
+                model.library.get_index(count["title"])
             except BookNameError:
                 print u"Old count: %s" % count["title"]
                 db.vstate.remove({"_id": count["_id"]})

+ 2 - 2
sefaria/export.py

@@ -67,7 +67,7 @@ def make_text(doc):
 
 	"""
 
-    index = model.get_index(doc["title"])
+    index = model.library.get_index(doc["title"])
     text = "\n".join([doc["title"], doc.get("heTitle", ""), doc["versionTitle"], doc["versionSource"]])    
     version = Version().load({'title': doc["title"], 'versionTitle': doc["versionTitle"], 'language': doc["language"]})	
 
@@ -153,7 +153,7 @@ def export_text(text):
 	"""
     print text["title"]
     try:
-        index = model.get_index(text["title"])
+        index = model.library.get_index(text["title"])
     except Exception as e:
         print "Skipping %s - %s" % (text["title"], e.message)
         return

+ 1 - 1
sefaria/helper/link.py

@@ -105,7 +105,7 @@ def rebuild_commentary_links(tref, user, **kwargs):
         oref = Ref(tref)
     except InputError:
         # Allow commentators alone, rebuild for each text we have
-        i = get_index(tref)
+        i = library.get_index(tref)
         for c in library.get_commentary_version_titles(i.title):
             rebuild_commentary_links(c, user, **kwargs)
         return

+ 1 - 1
sefaria/helper/text.py

@@ -18,7 +18,7 @@ def add_spelling(category, old, new, lang="en"):
     """
     indxs = library.get_indexes_in_category(category)
     for ind in indxs:
-        i = get_index(ind)
+        i = library.get_index(ind)
         print
         assert isinstance(i, Index)
         schema = i.nodes

+ 1 - 1
sefaria/model/count.py

@@ -35,7 +35,7 @@ class Count(abst.AbstractMongoRecord):
 
         if getattr(self, "title", None):
             try:
-                indx = text.get_index(self.title)
+                indx = text.library.get_index(self.title)
                 attrs = indx.contents()
                 #del attrs["_id"]
                 self.index_attr_keys = attrs.keys()

+ 2 - 2
sefaria/model/tests/text_test.py

@@ -74,14 +74,14 @@ def test_index_methods():
 
 
 def test_get_index():
-    r = model.get_index("Rashi on Exodus")
+    r = model.library.get_index("Rashi on Exodus")
     assert isinstance(r, model.CommentaryIndex)
     assert u'Rashi on Exodus' == r.title
     assert u'Rashi on Exodus' in r.titleVariants
     assert u'Rashi' not in r.titleVariants
     assert u'Exodus' not in r.titleVariants
 
-    r = model.get_index("Exodus")
+    r = model.library.get_index("Exodus")
     assert isinstance(r, model.Index)
     assert r.title == u'Exodus'
 

+ 1 - 1
sefaria/model/tests/vstate_test.py

@@ -8,7 +8,7 @@ class Test_VState(object):
     def test_integrity(self):
         titles = ["Exodus", "Shabbat", "Rashi on Exodus", "Rashi on Genesis", "Rashi on Shabbat"]
         for title in titles:
-            index = get_index(title)
+            index = library.get_index(title)
             vs = VersionState(index)
             assert getattr(vs, "title")
             assert getattr(vs, "content")

+ 64 - 69
sefaria/model/text.py

@@ -550,7 +550,7 @@ class Index(abst.AbstractMongoRecord, AbstractIndex):
             on_split = self.get_title().split(" on ")
             if len(on_split) == 2:
                 try:
-                    i = get_index(on_split[1])
+                    i = library.get_index(on_split[1])
                     if getattr(i, "order", None):
                         toc_contents_dict["order"] = i.order
                 except BookNameError:
@@ -591,7 +591,7 @@ class CommentaryIndex(AbstractIndex):
         if not self.c_index:
             raise BookNameError(u"No commentator named '{}'.".format(commentator_name))
 
-        self.b_index = get_index(book_name)
+        self.b_index = library.get_index(book_name)
 
         if not self.b_index:
             raise BookNameError(u"No book named '{}'.".format(book_name))
@@ -740,50 +740,10 @@ class CommentaryIndex(AbstractIndex):
 
         return attrs
 
-
+# Deprecated
 def get_index(bookname):
-    """
-    Factory - returns either an :class:`Index` object or a :class:`CommentaryIndex` object
-
-    :param string bookname: Name of the book or commentary on book.
-    :return:
-    """
-    # look for result in indices cache
-    if not bookname:
-        raise BookNameError("No book provided.")
-
-    cached_result = scache.get_index(bookname)
-    if cached_result:
-        return cached_result
-
-    bookname = (bookname[0].upper() + bookname[1:]).replace("_", " ")  #todo: factor out method
-
-    #todo: cache
-    node = library.get_schema_node(bookname)
-    if node:
-        i = node.index
-        scache.set_index(bookname, i)
-        return i
-
-    # "commenter" on "book"
-    # todo: handle hebrew x on y format (do we need this?)
-    pattern = r'(?P<commentor>.*) on (?P<book>.*)'
-    m = regex.match(pattern, bookname)
-    if m:
-        i = CommentaryIndex(m.group('commentor'), m.group('book'))
-        scache.set_index(bookname, i)
-        return i
-
-    #simple commentary record
-    c_index = Index().load({
-            "titleVariants": bookname,
-            "categories.0": "Commentary"
-        })
-    if c_index:
-        return c_index
-
-    raise BookNameError(u"No book named '{}'.".format(bookname))
-
+    logger.warning("Use of deprecated function: get_index()")
+    return library.get_index(bookname)
 
 
 """
@@ -933,7 +893,7 @@ class Version(abst.AbstractMongoRecord, AbstractTextRecord, AbstractSchemaConten
         pass
 
     def get_index(self):
-        return get_index(self.title)
+        return library.get_index(self.title)
 
     def first_section_ref(self):
         """
@@ -1812,7 +1772,7 @@ class Ref(object):
             if match:
                 title = match.group('title')
                 on_node = library.get_schema_node(match.group('commentee'))  # May be SchemaNode or JaggedArrayNode
-                self.index = get_index(match.group('commentor') + " on " + on_node.index.title)
+                self.index = library.get_index(match.group('commentor') + " on " + on_node.index.title)
                 self.index_node = self.index.nodes.title_dict(self._lang).get(title)
                 self.book = self.index_node.full_title("en")
                 if not self.index_node:
@@ -3278,16 +3238,16 @@ class Library(object):
         self._index_ref_map = {}
 
         # Maps, keyed by language, from index key to array of titles
-        self._index_title_maps = {}
+        self._index_title_maps = {lang:{} for lang in self.langs}
 
         # Maps, keyed by language, from titles to schema nodes
-        self._title_node_maps = {}
+        self._title_node_maps = {lang:{} for lang in self.langs}
 
         # Maps, keyed by language, from index key to array of commentary titles
-        self._index_title_commentary_maps = {}
+        self._index_title_commentary_maps = {lang:{} for lang in self.langs}
 
         # Maps, keyed by language, from titles to simple and commentary schema nodes
-        self._title_node_with_commentary_maps = {}
+        self._title_node_with_commentary_maps = {lang:{} for lang in self.langs}
 
         # Lists of full titles, keys are string generated from a combination of language code, "commentators", "commentary", and "terms".  See method `full_title_list()`
         self._full_title_lists = {}
@@ -3300,11 +3260,55 @@ class Library(object):
         self._title_regexes = {}
 
         # Maps, keyed by language, from term names to text refs
-        self._term_ref_maps = {}
+        self._term_ref_maps = {lang:{} for lang in self.langs}
+
+        # Map from index title to index object
+        self._indexes = {}
 
         # old local cache
         self.local_cache = {}
 
+
+    def get_index(self, bookname):
+        """
+        Factory - returns either an :class:`Index` object or a :class:`CommentaryIndex` object
+
+        :param string bookname: Name of the book or commentary on book.
+        :return:
+        """
+        # look for result in indices cache
+        if not bookname:
+            raise BookNameError("No book provided.")
+
+        indx = self._indexes.get(bookname)
+        if not indx:
+            bookname = (bookname[0].upper() + bookname[1:]).replace("_", " ")  #todo: factor out method
+
+            #todo: cache
+            node = self.get_schema_node(bookname)
+            if node:
+                indx = node.index
+            else:
+                # "commenter" on "book"
+                # todo: handle hebrew x on y format (do we need this?)
+                pattern = r'(?P<commentor>.*) on (?P<book>.*)'
+                m = regex.match(pattern, bookname)
+                if m:
+                    indx = CommentaryIndex(m.group('commentor'), m.group('book'))
+                else:
+                    #simple commentary record
+                    indx = Index().load({
+                            "titleVariants": bookname,
+                            "categories.0": "Commentary"
+                        })
+
+            if not indx:
+                raise BookNameError(u"No book named '{}'.".format(bookname))
+
+            self._indexes[bookname] = indx
+
+        return indx
+
     def add_index_record(self, index_title = None, index_object = None, rebuild = True):
         """
         Update library title dictionaries and caches with information from provided index.
@@ -3315,7 +3319,7 @@ class Library(object):
         :return:
         """
         if index_title:
-            index_object = get_index(index_title)
+            index_object = self.get_index(index_title)
         assert index_object, "Library.add_index_record called without index"
 
         #//TODO: mark for commentary refactor
@@ -3385,38 +3389,29 @@ class Library(object):
         self._title_regex_strings = {}
         self._title_regexes = {}
 
-
     def build_all_title_node_dicts(self):
         # Rework get_index_forest() code here to only run once
 
         # simple texts
         forest = [i.nodes for i in IndexSet() if not i.is_commentary()]
-        title_dicts = {lang : {} for lang in self.langs}
+        self._title_node_maps = {lang : {} for lang in self.langs}
         for tree in forest:
             try:
                 for lang in self.langs:
-                    title_dicts[lang].update(tree.title_dict(lang))
+                    self._title_node_maps[lang].update(tree.title_dict(lang))
             except IndexSchemaError as e:
                 logger.error(u"Error in generating title node dictionary: {}".format(e))
-        for lang in self.langs:
-            key = "title_node_dict_" + lang
-            scache.set_cache_elem(key, title_dicts[lang])
-            self.local_cache[key] = title_dicts[lang]
 
         # commentary
-        commentary_forest = [get_index(i).nodes for i in self.get_commentary_version_titles()]
-        c_title_dicts = { lang: title_dicts[lang].copy() for lang in self.langs }
+        commentary_forest = [self.get_index(i).nodes for i in self.get_commentary_version_titles()]
+        self._title_node_with_commentary_maps = { lang: self._title_node_maps[lang].copy() for lang in self.langs }
 
         for tree in commentary_forest:
             try:
                 for lang in self.langs:
-                    c_title_dicts[lang].update(tree.title_dict(lang))
+                    self._title_node_with_commentary_map[lang].update(tree.title_dict(lang))
             except IndexSchemaError as e:
                 logger.error(u"Error in generating title node dictionary: {}".format(e))
-        for lang in self.langs:
-            comm_key = "title_node_dict_" + lang + "_commentary"
-            scache.set_cache_elem(comm_key, c_title_dicts[lang])
-            self.local_cache[comm_key] = c_title_dicts[lang]
 
     #todo: the for_js path here does not appear to be in use.
     def all_titles_regex_string(self, lang="en", commentary=False, with_commentary=False, with_terms=False): #, for_js=False):
@@ -3568,7 +3563,7 @@ class Library(object):
             ctitles = self.get_commentary_version_titles()
             for title in ctitles:
                 try:
-                    i = get_index(title)
+                    i = self.get_index(title)
                     root_nodes.append(i.nodes)
 
                 # TEMPORARY - filter out complex texts
@@ -3591,7 +3586,7 @@ class Library(object):
         # title_dict = self.local_cache.get(key)
 
         #//TODO: mark for commentary refactor
-        title_dict = self._title_node_maps[lang] if with_commentary else self._title_node_with_commentary_maps[lang]
+        title_dict = self._title_node_maps.get(lang) if with_commentary else self._title_node_with_commentary_maps.get(lang)
 
         #if not title_dict:
         #    title_dict = scache.get_cache_elem(key)

+ 3 - 3
sefaria/model/version_state.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 from . import abstract as abst
 from . import text
 from . import link
-from text import VersionSet, AbstractIndex, AbstractSchemaContent, IndexSet, library, get_index, Ref
+from text import VersionSet, AbstractIndex, AbstractSchemaContent, IndexSet, library, Ref
 from sefaria.datatype.jagged_array import JaggedTextArray, JaggedIntArray
 from sefaria.system.exceptions import InputError, BookNameError
 from sefaria.system.cache import delete_template_cache
@@ -104,14 +104,14 @@ class VersionState(abst.AbstractMongoRecord, AbstractSchemaContent):
         if not index:  # so that basic model tests can run
             if getattr(self, "title", None):
                 try:
-                    self.index = get_index(self.title)
+                    self.index = library.get_index(self.title)
                 except BookNameError as e:
                     logger.warning(u"Failed to load Index for VersionState - {}: {} (Normal on Index name change)".format(self.title, e))
             return
 
         if not isinstance(index, AbstractIndex):
             try:
-                index = get_index(index)
+                index = library.get_index(index)
             except BookNameError as e:
                 logger.warning("Failed to load Index for VersionState {}: {}".format(index, e))
 

+ 2 - 2
sefaria/summaries.py

@@ -201,7 +201,7 @@ def update_table_of_contents():
     for c in commentary_texts:
 
         try:
-            i = get_index(c)
+            i = library.get_index(c)
         except BookNameError:
             continue
 
@@ -258,7 +258,7 @@ def update_summaries_on_change(bookname, old_ref=None, recount=True):
     Update text summary docs to account for change or insertion of 'text'
     * recount - whether or not to perform a new count of available text
     """
-    index = get_index(bookname)
+    index = library.get_index(bookname)
 
     indx_dict = index.toc_contents()
 

+ 2 - 2
sefaria/utils/hebrew.py

@@ -509,9 +509,9 @@ def hebrew_term(s):
 
 	# If s is a text title, look for a stored Hebrew title
 	try:
-		from sefaria.model import get_index, IndexSet
+		from sefaria.model import library, IndexSet
 		from sefaria.system.exceptions import BookNameError
-		i = get_index(s)
+		i = library.get_index(s)
 		return i.get_title("he")
 	except BookNameError:
 		pass

+ 1 - 1
sefaria/views.py

@@ -268,7 +268,7 @@ def del_cached_elem(request, title):
 def reset_counts(request, title=None):
     if title:
         try:
-            i  = model.get_index(title)
+            i  = model.library.get_index(title)
         except:
             return HttpResponseRedirect("/dashboard?m=Unknown-Book")
         vs = model.VersionState(index=i)