Browse Source

Merge remote-tracking branch 'origin/reduce-server-startup' into explore

# Conflicts:
#	locale/en/LC_MESSAGES/django.po
#	locale/he/LC_MESSAGES/django.mo
#	locale/he/LC_MESSAGES/django.po
#	sefaria/model/link.py
Ephraim Damboritz 3 months ago
parent
commit
00995be96d
53 changed files with 1495 additions and 620 deletions
  1. 1 1
      build/prime_cache.sh
  2. BIN
      locale/en/LC_MESSAGES/django.mo
  3. 125 57
      locale/en/LC_MESSAGES/django.po
  4. 139 62
      locale/he/LC_MESSAGES/django.po
  5. 67 53
      package-lock.json
  6. 73 75
      reader/views.py
  7. 2 1
      requirements.txt
  8. 0 0
      scripts/invalid refs.txt
  9. 5 1
      scripts/metrics.py
  10. 63 0
      scripts/mishnah_yomit.py
  11. 45 0
      scripts/netzach_yisrael.py
  12. 42 0
      scripts/regenerate_explorer_data.py
  13. 1 1
      scripts/rewrite_rashis_in_tanchuma.py
  14. 20 0
      scripts/sheet_tag_usage_count.py
  15. 47 0
      scripts/sheets_with_non_library_content.py
  16. 53 0
      scripts/top_pagerank_parsha.py
  17. 5 1
      sefaria/client/util.py
  18. 3 1
      sefaria/helper/schema.py
  19. 145 0
      sefaria/helper/search.py
  20. 14 1
      sefaria/helper/tests/schema_test.py
  21. 40 0
      sefaria/helper/tests/search_test.py
  22. 44 14
      sefaria/model/autospell.py
  23. 1 9
      sefaria/model/link.py
  24. 3 17
      sefaria/model/notification.py
  25. 22 1
      sefaria/model/text.py
  26. 4 2
      sefaria/model/user_profile.py
  27. 28 3
      sefaria/pagesheetrank.py
  28. 4 6
      sefaria/search.py
  29. 2 2
      sefaria/sheets.py
  30. 32 10
      sefaria/system/cache.py
  31. 13 0
      sefaria/system/decorators.py
  32. 5 4
      sefaria/urls.py
  33. 28 18
      sefaria/utils/util.py
  34. 35 6
      sefaria/views.py
  35. 21 27
      static/css/s2.css
  36. 6 5
      static/js/ConnectionsPanel.jsx
  37. 28 18
      static/js/DictionarySearch.jsx
  38. 47 32
      static/js/LexiconBox.jsx
  39. 10 3
      static/js/Misc.jsx
  40. 6 2
      static/js/ReaderPanel.jsx
  41. 0 1
      static/js/ReaderTextTableOfContents.jsx
  42. 4 6
      static/js/SearchResultList.jsx
  43. 2 2
      static/js/Sheet.jsx
  44. 2 2
      static/js/UserHistoryPanel.jsx
  45. 22 140
      static/js/sefaria/search.js
  46. 2 4
      static/js/sefaria/searchState.js
  47. 8 19
      static/js/sefaria/sefaria.js
  48. 0 1
      templates/js/data.js
  49. 5 9
      templates/sheets.html
  50. 1 1
      templates/static/newsletter.html
  51. 0 1
      templates/static/supporters.html
  52. 219 0
      templates/static/testimonials.html
  53. 1 1
      templates/talmudic_relationships.html

+ 1 - 1
build/prime_cache.sh

@@ -7,8 +7,8 @@ curl -s http://localhost/api/name/stam > /dev/null &
 curl -s http://localhost/api/name/stam?ref_only=1 > /dev/null &
 curl -s http://localhost/api/name/%D7%A1%D7%AA%D7%9D?ref_only=1 > /dev/null &
 curl -s http://localhost/api/name/%D7%A1%D7%AA%D7%9D > /dev/null &
-curl -s http://localhost/api/counts/links/Tanach/Bavli > /dev/null &
 curl -s http://localhost/api/preview/Zohar > /dev/null &
+#curl -s http://localhost/api/counts/links/Tanach/Bavli > /dev/null &
 #curl -s http://localhost/api/texts/version-status/tree/ > /dev/null &
 #curl -s http://localhost/api/texts/version-status/tree/he > /dev/null &
 #curl -s http://localhost/api/texts/version-status/tree/en > /dev/null &

BIN
locale/en/LC_MESSAGES/django.mo


+ 125 - 57
locale/en/LC_MESSAGES/django.po

@@ -3,12 +3,11 @@
 # This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
-#, fuzzy
 msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-12-31 11:21-0800\n"
+"POT-Creation-Date: 2019-01-10 11:06+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -29,89 +28,89 @@ msgstr ""
 msgid "Bilingual"
 msgstr ""
 
-#: reader/views.py:465
+#: reader/views.py:467
 #, python-format
 msgid "Read the text of %(book)s online with commentaries and connections."
 msgstr ""
 
-#: reader/views.py:484
+#: reader/views.py:486
 msgid "Explore 3,000 years of Jewish texts in Hebrew and English translation."
 msgstr ""
 
-#: reader/views.py:531 reader/views.py:684
+#: reader/views.py:533 reader/views.py:708
 msgid " | Sefaria"
 msgstr ""
 
-#: reader/views.py:532
+#: reader/views.py:534
 #, python-format
 msgid "Read %(categories)s texts online with commentaries and connections."
 msgstr ""
 
-#: reader/views.py:535
+#: reader/views.py:537
 msgid "Recently Viewed"
 msgstr ""
 
-#: reader/views.py:536
+#: reader/views.py:538
 msgid "Texts that you've recently viewed on Sefaria."
 msgstr ""
 
-#: reader/views.py:583
+#: reader/views.py:607
 msgid "Sefaria Search"
 msgstr ""
 
-#: reader/views.py:584
+#: reader/views.py:608
 msgid "Search 3,000 years of Jewish texts in Hebrew and English translation."
 msgstr ""
 
-#: reader/views.py:600
+#: reader/views.py:624
 msgid "Sefaria Source Sheets"
 msgstr ""
 
-#: reader/views.py:601
+#: reader/views.py:625
 msgid ""
 "Explore thousands of public Source Sheets and use our Source Sheet Builder "
 "to create your own online."
 msgstr ""
 
-#: reader/views.py:629 reader/views.py:636 reader/views.py:643
+#: reader/views.py:653 reader/views.py:660 reader/views.py:667
 #: templates/groups.html:8
 msgid "Sefaria Groups"
 msgstr ""
 
-#: reader/views.py:649
+#: reader/views.py:673
 msgid "My Notes on Sefaria"
 msgstr ""
 
-#: reader/views.py:670
+#: reader/views.py:694
 msgid "My Source Sheets | Sefaria Source Sheets"
 msgstr ""
 
-#: reader/views.py:671
+#: reader/views.py:695
 msgid "My Sources Sheets on Sefaria, both private and public."
 msgstr ""
 
-#: reader/views.py:678
+#: reader/views.py:702
 msgid "Public Source Sheets | Sefaria Source Sheets"
 msgstr ""
 
-#: reader/views.py:679
+#: reader/views.py:703
 msgid ""
 "Explore thousands of public Source Sheets drawing on Sefaria's library of "
 "Jewish texts."
 msgstr ""
 
-#: reader/views.py:685
+#: reader/views.py:709
 #, python-format
 msgid ""
 "Public Source Sheets on tagged with \"%(tag)s\", drawing from Sefaria's "
 "library of Jewish texts."
 msgstr ""
 
-#: reader/views.py:782
+#: reader/views.py:806
 msgid "Topics"
 msgstr ""
 
-#: reader/views.py:782 templates/person.html:6 templates/person.html:8
+#: reader/views.py:806 templates/person.html:6 templates/person.html:8
 #: templates/registration/password_reset_complete.html:3
 #: templates/registration/password_reset_confirm.html:4
 #: templates/registration/password_reset_done.html:3
@@ -124,59 +123,75 @@ msgstr ""
 msgid "Sefaria"
 msgstr ""
 
-#: reader/views.py:783
+#: reader/views.py:807
 msgid "Explore Jewish Texts by Topic on Sefaria"
 msgstr ""
 
-#: reader/views.py:841
+#: reader/views.py:865
 msgid "The Sefaria Library"
 msgstr ""
 
-#: reader/views.py:842
+#: reader/views.py:866
 msgid ""
 "Browse 1,000s of Jewish texts in the Sefaria Library by category and title."
 msgstr ""
 
-#: reader/views.py:847
+#: reader/views.py:872
+msgid "My Saved Content"
+msgstr ""
+
+#: reader/views.py:873
+msgid "See your saved content on Sefaria"
+msgstr ""
+
+#: reader/views.py:879
+msgid "My User History"
+msgstr ""
+
+#: reader/views.py:880
+msgid "See your user history on Sefaria"
+msgstr ""
+
+#: reader/views.py:886
 msgid "New Additions to the Sefaria Library"
 msgstr ""
 
-#: reader/views.py:848
+#: reader/views.py:887
 msgid ""
 "See texts, translations and connections that have been recently added to "
 "Sefaria."
 msgstr ""
 
-#: reader/views.py:854
+#: reader/views.py:893
 msgid "Sefaria Account"
 msgstr ""
 
-#: reader/views.py:862
+#: reader/views.py:901
 msgid "Sefaria Notifications"
 msgstr ""
 
-#: reader/views.py:869
+#: reader/views.py:908
 msgid "Moderator Tools"
 msgstr ""
 
-#: reader/views.py:886
+#: reader/views.py:925
 msgid "Extended Notes"
 msgstr ""
 
-#: reader/views.py:936
+#: reader/views.py:975
 msgid "Texts"
 msgstr ""
 
-#: reader/views.py:2860
+#: reader/views.py:2869 reader/views.py:2903
 msgid "You must be logged in to update your profile."
 msgstr ""
 
-#: reader/views.py:3645 reader/views.py:3648
+#: reader/views.py:3668 reader/views.py:3671
 #, python-format
 msgid "Learn about %(name)s - works written, biographies, dates and more."
 msgstr ""
 
-#: sefaria/forms.py:26 sefaria/forms.py:31 sefaria/forms.py:103
+#: sefaria/forms.py:26 sefaria/forms.py:31 sefaria/forms.py:106
 #: templates/static/aramaic-translation-contest.html:140
 #: templates/static/educators.html:359 templates/static/home.html:362
 #: templates/static/newsletter.html:24
@@ -217,6 +232,59 @@ msgstr ""
 msgid "Public Groups are required to have at least 3 public sheets."
 msgstr ""
 
+#: sefaria/utils/util.py:15
+#, python-format
+msgid "%d year"
+msgid_plural "%d years"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:16
+#, python-format
+msgid "%d month"
+msgid_plural "%d months"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:17
+#, python-format
+msgid "%d week"
+msgid_plural "%d weeks"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:18
+#, python-format
+msgid "%d day"
+msgid_plural "%d days"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:19
+#, python-format
+msgid "%d hour"
+msgid_plural "%d hours"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:20
+#, python-format
+msgid "%d minute"
+msgid_plural "%d minutes"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:21
+#, python-format
+msgid "%d second"
+msgid_plural "%d seconds"
+msgstr[0] ""
+msgstr[1] ""
+
+#: sefaria/utils/util.py:52
+msgid "now"
+msgstr ""
+
 #: sefaria/views.py:125 sefaria/views.py:157
 #: templates/static/aramaic-translation-contest.html:189
 #: templates/static/educators.html:389 templates/static/educators.html:411
@@ -514,27 +582,12 @@ msgid ""
 "Link Explorer - Visualized Connections Between Talmud and Tanakh | Sefaria"
 msgstr ""
 
-#: templates/explore.html:4
-#, python-format
-msgid ""
-"Link Explorer - Visualized Connections Between %(bottomCatTitle)s and "
-"%(topCatTitle)s | Sefaria"
-msgstr ""
-
 #: templates/explore.html:6
 msgid ""
 "Explore over 30,000 examples of the Talmud quoting a verse from Tanakh. Zoom "
 "in to particular books either above or below to explore deeper."
 msgstr ""
 
-#: templates/explore.html:6
-#, python-format
-msgid ""
-"Visually explore quotations and connections between %(bottomCatTitle)s and "
-"%(topCatTitle)s. Zoom in to particular books either above or below to "
-"explore deeper."
-msgstr ""
-
 #: templates/leaderboard.html:4
 msgid "Top Contributors | Sefaria"
 msgstr ""
@@ -679,16 +732,19 @@ msgid "Password Reset Link Sent"
 msgstr ""
 
 #: templates/registration/password_reset_email.html:4
+#: templates/registration/password_reset_email.txt:3
 msgid ""
 "A request has been made to reset the password for this email address on "
 "Sefaria.org."
 msgstr ""
 
 #: templates/registration/password_reset_email.html:6
+#: templates/registration/password_reset_email.txt:5
 msgid "Click this link, or copy it to your browser, to choose a new password:"
 msgstr ""
 
 #: templates/registration/password_reset_email.html:12
+#: templates/registration/password_reset_email.txt:9
 msgid "If you did not request this reset, you can safely ignore this email."
 msgstr ""
 
@@ -761,35 +817,39 @@ msgstr ""
 msgid "Search for a Text or Commentator"
 msgstr ""
 
-#: templates/sheets.html:920
+#: templates/sheets.html:819
+msgid "Paste URL"
+msgstr ""
+
+#: templates/sheets.html:915
 msgid "Made with the Sefaria Source Sheet Builder"
 msgstr ""
 
-#: templates/sheets.html:943
+#: templates/sheets.html:938
 msgid "Create New"
 msgstr ""
 
-#: templates/sheets.html:1017
+#: templates/sheets.html:1012
 msgid "Write a summary to help others understand your sheet..."
 msgstr ""
 
-#: templates/sheets.html:1053
+#: templates/sheets.html:1048
 msgid "None"
 msgstr ""
 
-#: templates/sheets.html:1087
+#: templates/sheets.html:1082
 msgid "View"
 msgstr ""
 
-#: templates/sheets.html:1090
+#: templates/sheets.html:1085
 msgid "Add"
 msgstr ""
 
-#: templates/sheets.html:1093
+#: templates/sheets.html:1088
 msgid "Edit"
 msgstr ""
 
-#: templates/sheets.html:1106
+#: templates/sheets.html:1101
 msgid ""
 "\n"
 "            Add <span class=\"sourceName\"></span> to a Source Sheet\n"
@@ -1012,6 +1072,14 @@ msgstr ""
 msgid "Watch a video about the story of Sefaria."
 msgstr ""
 
+#: templates/static/torah-tab.html:5
+msgid "Sefaria Torah Tab"
+msgstr ""
+
+#: templates/static/torah-tab.html:7
+msgid "Install the free Sefaria Torah Tab for your web browser."
+msgstr ""
+
 #: templates/static/visualizations.html:4
 msgid "Visualizations"
 msgstr ""

+ 139 - 62
locale/he/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-12-31 11:21-0800\n"
+"POT-Creation-Date: 2019-01-10 11:06+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -29,47 +29,47 @@ msgstr "עברית"
 msgid "Bilingual"
 msgstr "דו לשוני"
 
-#: reader/views.py:465
+#: reader/views.py:467
 #, python-format
 msgid "Read the text of %(book)s online with commentaries and connections."
 msgstr "%(book)s מקוון בליווי מפרשים וקשרים לטקסטים אחרים"
 
-#: reader/views.py:484
+#: reader/views.py:486
 msgid "Explore 3,000 years of Jewish texts in Hebrew and English translation."
 msgstr "צאו למסע בעקבות 3000 שנה של טקסטים יהודיים, בעברית ובתרגום לאנגלית"
 
-#: reader/views.py:531 reader/views.py:684
+#: reader/views.py:533 reader/views.py:708
 msgid " | Sefaria"
 msgstr " | ספריא"
 
-#: reader/views.py:532
+#: reader/views.py:534
 #, python-format
 msgid "Read %(categories)s texts online with commentaries and connections."
 msgstr "ספרי %(categories)s מקוונים"
 
-#: reader/views.py:535
+#: reader/views.py:537
 msgid "Recently Viewed"
 msgstr "נצפו לאחרונה"
 
-#: reader/views.py:536
+#: reader/views.py:538
 msgid "Texts that you've recently viewed on Sefaria."
 msgstr "ספרים שנצפו לאחרונה בספריא"
 
-#: reader/views.py:583
+#: reader/views.py:607
 msgid "Sefaria Search"
 msgstr "חיפוש בספריא"
 
-#: reader/views.py:584
+#: reader/views.py:608
 msgid "Search 3,000 years of Jewish texts in Hebrew and English translation."
 msgstr ""
 "חפשו בין אוצר המקורות הכולל 3000 שנה של כתבים יהודיים. בעברית ובתרגום "
 "לאנגלית."
 
-#: reader/views.py:600
+#: reader/views.py:624
 msgid "Sefaria Source Sheets"
 msgstr "דפי המקורות בספריא"
 
-#: reader/views.py:601
+#: reader/views.py:625
 msgid ""
 "Explore thousands of public Source Sheets and use our Source Sheet Builder "
 "to create your own online."
@@ -77,34 +77,34 @@ msgstr ""
 "סיירו בין אלפי דפי מקורות פומביים והשתמשו בבונה דפי המקורות בכדי ליצור דף "
 "מקורות משלכם."
 
-#: reader/views.py:629 reader/views.py:636 reader/views.py:643
+#: reader/views.py:653 reader/views.py:660 reader/views.py:667
 #: templates/groups.html:8
 msgid "Sefaria Groups"
 msgstr "קבוצות בספריא"
 
-#: reader/views.py:649
+#: reader/views.py:673
 msgid "My Notes on Sefaria"
 msgstr "הרשומות שלי בספריא"
 
-#: reader/views.py:670
+#: reader/views.py:694
 msgid "My Source Sheets | Sefaria Source Sheets"
 msgstr "דפי המקורות שלי | דפי מקורות בספריא"
 
-#: reader/views.py:671
+#: reader/views.py:695
 msgid "My Sources Sheets on Sefaria, both private and public."
 msgstr "דפי המקורות שלי בספריא. פרטיים ופומביים"
 
-#: reader/views.py:678
+#: reader/views.py:702
 msgid "Public Source Sheets | Sefaria Source Sheets"
 msgstr "דפי המקורות פומביים | דפי מקורות בספריא"
 
-#: reader/views.py:679
+#: reader/views.py:703
 msgid ""
 "Explore thousands of public Source Sheets drawing on Sefaria's library of "
 "Jewish texts."
 msgstr "סיירו בין אלפי דפי מקורות פומביים השאובים מאוצר הספרות של ספריא"
 
-#: reader/views.py:685
+#: reader/views.py:709
 #, python-format
 msgid ""
 "Public Source Sheets on tagged with \"%(tag)s\", drawing from Sefaria's "
@@ -113,11 +113,11 @@ msgstr ""
 "דפי מקורות פומביים המתוייגים ב\"%(tag)s\" השאובים מהמקורות שבאוצר הספרות "
 "בספריא."
 
-#: reader/views.py:782
+#: reader/views.py:806
 msgid "Topics"
 msgstr "נושאים"
 
-#: reader/views.py:782 templates/person.html:6 templates/person.html:8
+#: reader/views.py:806 templates/person.html:6 templates/person.html:8
 #: templates/registration/password_reset_complete.html:3
 #: templates/registration/password_reset_confirm.html:4
 #: templates/registration/password_reset_done.html:3
@@ -130,61 +130,78 @@ msgstr "נושאים"
 msgid "Sefaria"
 msgstr "ספריא"
 
-#: reader/views.py:783
+#: reader/views.py:807
 msgid "Explore Jewish Texts by Topic on Sefaria"
 msgstr "מקורות לפי נושא בספריא"
 
-#: reader/views.py:841
+#: reader/views.py:865
 msgid "The Sefaria Library"
 msgstr "תוכן העניינים של ספריא"
 
-#: reader/views.py:842
+#: reader/views.py:866
 msgid ""
 "Browse 1,000s of Jewish texts in the Sefaria Library by category and title."
 msgstr ""
 "גלשו בתוככי אלפי ספרי אוצר הספרות היהודית לדורותיה. מצאו ספרים לפי שם או "
 "סוגה."
 
-#: reader/views.py:847
+#: reader/views.py:872
+msgid "My Saved Content"
+msgstr "התוכן השמור שלי"
+
+#: reader/views.py:873
+msgid "See your saved content on Sefaria"
+msgstr "צפייה במקורות ודפים שמורים בספריא"
+
+#: reader/views.py:879
+msgid "My User History"
+msgstr "היסטורית קריאה שלי"
+
+#: reader/views.py:880
+#| msgid "Watch a video about the story of Sefaria."
+msgid "See your user history on Sefaria"
+msgstr "צפייה בהיסטורית הגלישה בספריא"
+
+#: reader/views.py:886
 msgid "New Additions to the Sefaria Library"
 msgstr "חידושים בארון הספרים של ספריא"
 
-#: reader/views.py:848
+#: reader/views.py:887
 msgid ""
 "See texts, translations and connections that have been recently added to "
 "Sefaria."
 msgstr "הודעות לגבי טקסטים, תרגומים וקישורים חדשים שנוספו לאחרונה לספריא"
 
-#: reader/views.py:854
+#: reader/views.py:893
 msgid "Sefaria Account"
 msgstr "חשבון בספריא"
 
-#: reader/views.py:862
+#: reader/views.py:901
 msgid "Sefaria Notifications"
 msgstr "הודעות בספריא"
 
-#: reader/views.py:869
+#: reader/views.py:908
 msgid "Moderator Tools"
 msgstr "כלי מנהלים"
 
-#: reader/views.py:886
+#: reader/views.py:925
 msgid "Extended Notes"
-msgstr ""
+msgstr "מידע מורחב"
 
-#: reader/views.py:936
+#: reader/views.py:975
 msgid "Texts"
 msgstr "טקסטים"
 
-#: reader/views.py:2860
+#: reader/views.py:2869 reader/views.py:2903
 msgid "You must be logged in to update your profile."
 msgstr "הנכם צריכים להיות מחוברים כדי לערוך את הפרופיל שלכם."
 
-#: reader/views.py:3645 reader/views.py:3648
+#: reader/views.py:3668 reader/views.py:3671
 #, python-format
 msgid "Learn about %(name)s - works written, biographies, dates and more."
 msgstr "אודות %(name)s. יצירות שחוברו, ביוגרפיה, תאריכים ועוד"
 
-#: sefaria/forms.py:26 sefaria/forms.py:31 sefaria/forms.py:103
+#: sefaria/forms.py:26 sefaria/forms.py:31 sefaria/forms.py:106
 #: templates/static/aramaic-translation-contest.html:140
 #: templates/static/educators.html:359 templates/static/home.html:362
 #: templates/static/newsletter.html:24
@@ -219,11 +236,64 @@ msgstr "משתמש עם כתובת האימייל לעיל כבר קיים"
 msgid ""
 "Public Groups are required to include a group image (a square image will "
 "work best)."
-msgstr ""
+msgstr "קבוצות פומביות חייבות לכלול תמונת קבוצה"
 
 #: sefaria/model/group.py:59
 msgid "Public Groups are required to have at least 3 public sheets."
-msgstr ""
+msgstr "קבוצות פומביות חייבות לכלול לפחות 3 דפי מקורות פומביים"
+
+#: sefaria/utils/util.py:15
+#, python-format
+msgid "%d year"
+msgid_plural "%d years"
+msgstr[0] "%d שנה"
+msgstr[1] "%d שנים"
+
+#: sefaria/utils/util.py:16
+#, python-format
+msgid "%d month"
+msgid_plural "%d months"
+msgstr[0] "%d חודש"
+msgstr[1] "%d חודשים"
+
+#: sefaria/utils/util.py:17
+#, python-format
+msgid "%d week"
+msgid_plural "%d weeks"
+msgstr[0] "%d שבוע"
+msgstr[1] "%d שבועות"
+
+#: sefaria/utils/util.py:18
+#, python-format
+msgid "%d day"
+msgid_plural "%d days"
+msgstr[0] "%d יום"
+msgstr[1] "%d ימים"
+
+#: sefaria/utils/util.py:19
+#, python-format
+msgid "%d hour"
+msgid_plural "%d hours"
+msgstr[0] "%d שעה"
+msgstr[1] "%d שעות"
+
+#: sefaria/utils/util.py:20
+#, python-format
+msgid "%d minute"
+msgid_plural "%d minutes"
+msgstr[0] "%d דקה"
+msgstr[1] "%d דקות"
+
+#: sefaria/utils/util.py:21
+#, python-format
+msgid "%d second"
+msgid_plural "%d seconds"
+msgstr[0] "%d שנייה"
+msgstr[1] "%d שניות"
+
+#: sefaria/utils/util.py:52
+msgid "now"
+msgstr "כעת"
 
 #: sefaria/views.py:125 sefaria/views.py:157
 #: templates/static/aramaic-translation-contest.html:189
@@ -683,10 +753,9 @@ msgid "Cancel"
 msgstr "ביטול"
 
 #: templates/registration/logged_out.html:4
-#, fuzzy
 #| msgid "Log in to Sefaria"
 msgid "Logged Out | Sefaria"
-msgstr "כניסה לחשבון בספריא"
+msgstr "מנותק | ספריא"
 
 #: templates/registration/login.html:4
 msgid "Log in to Sefaria"
@@ -713,6 +782,7 @@ msgid "Password Reset Link Sent"
 msgstr "קישור לאיפוס סיסמא נשלח"
 
 #: templates/registration/password_reset_email.html:4
+#: templates/registration/password_reset_email.txt:3
 msgid ""
 "A request has been made to reset the password for this email address on "
 "Sefaria.org."
@@ -720,11 +790,13 @@ msgstr ""
 "בקשה התקבלה במערכת לאיפוס ססמת משתמש בעל כתובת מייל זו באתר sefaria.org.il"
 
 #: templates/registration/password_reset_email.html:6
+#: templates/registration/password_reset_email.txt:5
 msgid "Click this link, or copy it to your browser, to choose a new password:"
 msgstr ""
 "לחצו כאן, או העתיקו והדביקו את הכתובת לדפדפך שלכם בכדי לבחור ססמה חדשה:"
 
 #: templates/registration/password_reset_email.html:12
+#: templates/registration/password_reset_email.txt:9
 msgid "If you did not request this reset, you can safely ignore this email."
 msgstr "אם לא שלחתם בקשה לאיפוס סיסמא, הנכם מוזמנים להתעלם מהמייל הזה"
 
@@ -753,16 +825,15 @@ msgstr ""
 "וכדי להיות בקשר עם משתמשים אחרים"
 
 #: templates/sefer_hachinukh_mitzvot.html:7
-#, fuzzy
 #| msgid "Visualizations"
 msgid "Sefer HaChinukh Visualization"
-msgstr "תרשימים גרפיים"
+msgstr "תרשימים גרפיים מספר החינוך"
 
 #: templates/sefer_hachinukh_mitzvot.html:9
 msgid ""
 "Explore how different categories of Mitzvot connect to one another through "
 "this parallel sets chart."
-msgstr ""
+msgstr "גרף המדגים כיצד קבוצות שונות של מצוות קשורות אחת לשניה"
 
 #: templates/sheets.html:6 templates/sheets_visual.html:4
 msgid "Sefaria Source Sheet Builder"
@@ -773,10 +844,9 @@ msgid "Start a New Sheet - Sefaria Source Sheet Builder"
 msgstr "יצירת דף מקורות חדש - בונה דפי המקורות של ספריא"
 
 #: templates/sheets.html:8 templates/sheets.html:9
-#, fuzzy
 #| msgid "Create your own source sheet with Sefaria's Source Sheet Builder."
 msgid "A source sheet created with Sefaria's Source Sheet Builder."
-msgstr "צרו דף מקורות משלכם בעזרת בונה דפי המקורות של ספריא"
+msgstr "דף מקורות בנוי בעזרת מחולל דפי המקורות של ספריא"
 
 #: templates/sheets.html:8 templates/sheets.html:9
 #: templates/sheets_visual.html:6
@@ -809,35 +879,39 @@ msgstr ""
 msgid "Search for a Text or Commentator"
 msgstr "הקלידו לחיפוש שם של טקסט או פרשן"
 
-#: templates/sheets.html:920
+#: templates/sheets.html:819
+msgid "Paste URL"
+msgstr "הדבק קישור"
+
+#: templates/sheets.html:915
 msgid "Made with the Sefaria Source Sheet Builder"
 msgstr "נבנה באמצעות בונה דפי המקורות בספריא"
 
-#: templates/sheets.html:943
+#: templates/sheets.html:938
 msgid "Create New"
 msgstr "יצירת חדש"
 
-#: templates/sheets.html:1017
+#: templates/sheets.html:1012
 msgid "Write a summary to help others understand your sheet..."
 msgstr "כתבו תקציר המסביר אודות דף המקורות שיצרתם..."
 
-#: templates/sheets.html:1053
+#: templates/sheets.html:1048
 msgid "None"
 msgstr "ללא"
 
-#: templates/sheets.html:1087
+#: templates/sheets.html:1082
 msgid "View"
 msgstr "לצפות"
 
-#: templates/sheets.html:1090
+#: templates/sheets.html:1085
 msgid "Add"
 msgstr "להוסיף"
 
-#: templates/sheets.html:1093
+#: templates/sheets.html:1088
 msgid "Edit"
 msgstr "לערוך"
 
-#: templates/sheets.html:1106
+#: templates/sheets.html:1101
 msgid ""
 "\n"
 "            Add <span class=\"sourceName\"></span> to a Source Sheet\n"
@@ -927,6 +1001,7 @@ msgid ""
 "directly to Sefaria and have not yet been published elsewhere in print or on "
 "the web."
 msgstr ""
+"טקסטים המציינים עמוד זה כמקור הטקסט נתרמו ישירות לספריא ולא פורסמו לפני כן במקומות אחרים בדפוס או בצורה מקוונת"
 
 #: templates/static/digitized-by-sefaria.html:4
 msgid "Texts Digitized by Sefaria"
@@ -1000,14 +1075,9 @@ msgid "Downland the free Sefaria Apps for iOS and Android."
 msgstr "הורידו את האפליקציות החינמיות של ספריא לאייפון ולאנדרואיד"
 
 #: templates/static/newsletter.html:4
-#, fuzzy
-#| msgid "Listed for Sefaria Users"
 msgid "Sign up for Sefaria's Newsletter"
-msgstr "מופיע למשתמשים רשומים"
+msgstr "הרשמה לרשימת התפוצה של ספריא"
 
-#: templates/static/newsletter.html:6
-msgid "Sign up for Sefaria's Newsletter."
-msgstr ""
 
 #: templates/static/privacy-policy.html:4
 msgid "Privacy Policy"
@@ -1069,14 +1139,12 @@ msgid "Terms of Use"
 msgstr "תנאי שימוש"
 
 #: templates/static/testimonials.html:4
-#, fuzzy
-#| msgid "Sefaria Metrics"
 msgid "Sefaria Testimonials"
-msgstr "סטטיסטיקות בספריא"
+msgstr "המלצות על ספריא"
 
 #: templates/static/testimonials.html:6
 msgid "Sefaria’s Impact – Reflections and Insights from Users"
-msgstr ""
+msgstr "ההשפעה של ספריא- הרהורים ותובנות משתמשים"
 
 #: templates/static/the-sefaria-story.html:4
 msgid "The Sefaria Story"
@@ -1086,6 +1154,14 @@ msgstr "תולדות ספריא"
 msgid "Watch a video about the story of Sefaria."
 msgstr "צפו בסרטון המתאר את קורות ספריא"
 
+#: templates/static/torah-tab.html:5
+msgid "Sefaria Torah Tab"
+msgstr "לשונית לימוד בספריא"
+
+#: templates/static/torah-tab.html:7
+msgid "Install the free Sefaria Torah Tab for your web browser."
+msgstr "התקינו את תוסף ספריא ללשונית לימוד"
+
 #: templates/static/visualizations.html:4
 msgid "Visualizations"
 msgstr "תרשימים גרפיים"
@@ -1124,7 +1200,8 @@ msgstr ""
 
 #: templates/talmudic_relationships.html:24
 msgid "Student-Teacher Relationships in the Talmud"
-msgstr ""
+msgstr "קשרי מורה-תלמיד בתלמוד"
+
 
 #: templates/talmudic_relationships.html:26
 msgid "Description goes here."

+ 67 - 53
package-lock.json

@@ -126,7 +126,7 @@
     },
     "array-flatten": {
       "version": "1.1.1",
-      "resolved": "http://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
     },
     "array-uniq": {
@@ -529,13 +529,13 @@
     },
     "babel-plugin-syntax-flow": {
       "version": "6.18.0",
-      "resolved": "http://registry.npmjs.org/babel-plugin-syntax-flow/-/babel-plugin-syntax-flow-6.18.0.tgz",
+      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-flow/-/babel-plugin-syntax-flow-6.18.0.tgz",
       "integrity": "sha1-TDqyCiryaqIM0lmVw5jE63AxDI0=",
       "dev": true
     },
     "babel-plugin-syntax-jsx": {
       "version": "6.18.0",
-      "resolved": "http://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz",
+      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz",
       "integrity": "sha1-CvMqmm4Tyno/1QaeYtew9Y0NiUY=",
       "dev": true
     },
@@ -1238,7 +1238,7 @@
     },
     "browserify-rsa": {
       "version": "4.0.1",
-      "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
       "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
       "dev": true,
       "requires": {
@@ -1272,7 +1272,7 @@
     },
     "buffer": {
       "version": "4.9.1",
-      "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
       "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
       "dev": true,
       "requires": {
@@ -1368,7 +1368,7 @@
     },
     "cheerio": {
       "version": "0.22.0",
-      "resolved": "http://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz",
       "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=",
       "requires": {
         "css-select": "~1.2.0",
@@ -1570,7 +1570,7 @@
     },
     "content-disposition": {
       "version": "0.5.2",
-      "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
       "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
     },
     "content-type": {
@@ -1706,7 +1706,7 @@
     },
     "css-select": {
       "version": "1.2.0",
-      "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
       "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
       "requires": {
         "boolbase": "~1.0.0",
@@ -1722,7 +1722,7 @@
     },
     "d": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/d/-/d-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
       "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
       "dev": true,
       "requires": {
@@ -1891,7 +1891,7 @@
       "dependencies": {
         "domelementtype": {
           "version": "1.1.3",
-          "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
           "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs="
         }
       }
@@ -2150,7 +2150,7 @@
     },
     "events": {
       "version": "1.1.1",
-      "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
       "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
       "dev": true
     },
@@ -2190,7 +2190,7 @@
     },
     "expand-range": {
       "version": "1.8.2",
-      "resolved": "http://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz",
+      "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz",
       "integrity": "sha1-opnv/TNf4nIeuujiV+x5ZE/IUzc=",
       "dev": true,
       "requires": {
@@ -2283,7 +2283,7 @@
     },
     "fast-deep-equal": {
       "version": "1.1.0",
-      "resolved": "http://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
       "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
     },
     "fast-json-stable-stringify": {
@@ -2453,12 +2453,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -2473,17 +2475,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -2600,7 +2605,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -2612,6 +2618,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -2626,6 +2633,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -2633,12 +2641,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -2657,6 +2667,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -2737,7 +2748,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -2749,6 +2761,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -2870,6 +2883,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -2949,7 +2963,7 @@
     },
     "get-stream": {
       "version": "3.0.0",
-      "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
       "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
       "dev": true
     },
@@ -3017,7 +3031,7 @@
     },
     "got": {
       "version": "6.7.1",
-      "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz",
+      "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
       "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
       "dev": true,
       "requires": {
@@ -3355,7 +3369,7 @@
     },
     "is-builtin-module": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
       "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
       "dev": true,
       "requires": {
@@ -3477,7 +3491,7 @@
     },
     "is-obj": {
       "version": "1.0.1",
-      "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
       "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=",
       "dev": true
     },
@@ -3576,7 +3590,7 @@
     },
     "jquery": {
       "version": "2.2.4",
-      "resolved": "http://registry.npmjs.org/jquery/-/jquery-2.2.4.tgz",
+      "resolved": "https://registry.npmjs.org/jquery/-/jquery-2.2.4.tgz",
       "integrity": "sha1-LInWiJterFIqfuoywUUhVZxsvwI="
     },
     "jquery-ui": {
@@ -3609,7 +3623,7 @@
     },
     "jsesc": {
       "version": "1.3.0",
-      "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz",
       "integrity": "sha1-RsP+yMGJKxKwgz25vHYiF226s0s=",
       "dev": true
     },
@@ -3636,7 +3650,7 @@
     },
     "json5": {
       "version": "0.5.1",
-      "resolved": "http://registry.npmjs.org/json5/-/json5-0.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz",
       "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=",
       "dev": true
     },
@@ -3686,7 +3700,7 @@
     },
     "load-json-file": {
       "version": "2.0.0",
-      "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
       "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
       "dev": true,
       "requires": {
@@ -3698,7 +3712,7 @@
       "dependencies": {
         "pify": {
           "version": "2.3.0",
-          "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
           "dev": true
         }
@@ -3923,7 +3937,7 @@
     },
     "media-typer": {
       "version": "0.3.0",
-      "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
     },
     "mem": {
@@ -4059,7 +4073,7 @@
     },
     "minimist": {
       "version": "0.0.8",
-      "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
       "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
       "dev": true
     },
@@ -4086,7 +4100,7 @@
     },
     "mkdirp": {
       "version": "0.5.1",
-      "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
       "dev": true,
       "requires": {
@@ -4772,7 +4786,7 @@
     },
     "os-homedir": {
       "version": "1.0.2",
-      "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
       "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
       "dev": true
     },
@@ -4789,7 +4803,7 @@
     },
     "os-tmpdir": {
       "version": "1.0.2",
-      "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
       "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
       "dev": true
     },
@@ -4899,7 +4913,7 @@
     },
     "path-browserify": {
       "version": "0.0.0",
-      "resolved": "http://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz",
       "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=",
       "dev": true
     },
@@ -4917,7 +4931,7 @@
     },
     "path-is-absolute": {
       "version": "1.0.1",
-      "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
       "dev": true
     },
@@ -4949,7 +4963,7 @@
       "dependencies": {
         "pify": {
           "version": "2.3.0",
-          "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
           "dev": true
         }
@@ -5637,7 +5651,7 @@
     },
     "regexpu-core": {
       "version": "2.0.0",
-      "resolved": "http://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz",
       "integrity": "sha1-SdA4g3uNz4v6W5pCE5k45uoq4kA=",
       "dev": true,
       "requires": {
@@ -5667,13 +5681,13 @@
     },
     "regjsgen": {
       "version": "0.2.0",
-      "resolved": "http://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz",
       "integrity": "sha1-bAFq3qxVT3WCP+N6wFuS1aTtsfc=",
       "dev": true
     },
     "regjsparser": {
       "version": "0.1.5",
-      "resolved": "http://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz",
+      "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz",
       "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=",
       "dev": true,
       "requires": {
@@ -5682,7 +5696,7 @@
       "dependencies": {
         "jsesc": {
           "version": "0.5.0",
-          "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz",
+          "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz",
           "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=",
           "dev": true
         }
@@ -5792,7 +5806,7 @@
     },
     "safe-regex": {
       "version": "1.1.0",
-      "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
       "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
       "dev": true,
       "requires": {
@@ -6221,7 +6235,7 @@
     },
     "stream-browserify": {
       "version": "2.0.1",
-      "resolved": "http://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
       "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=",
       "dev": true,
       "requires": {
@@ -6337,7 +6351,7 @@
     },
     "strip-ansi": {
       "version": "3.0.1",
-      "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
       "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
       "dev": true,
       "requires": {
@@ -6352,7 +6366,7 @@
     },
     "strip-eof": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
       "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
       "dev": true
     },
@@ -6497,7 +6511,7 @@
     },
     "tty-browserify": {
       "version": "0.0.0",
-      "resolved": "http://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
       "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=",
       "dev": true
     },
@@ -6548,7 +6562,7 @@
         },
         "yargs": {
           "version": "3.10.0",
-          "resolved": "http://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
           "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
           "dev": true,
           "requires": {
@@ -6833,7 +6847,7 @@
     },
     "vm-browserify": {
       "version": "0.0.4",
-      "resolved": "http://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz",
+      "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz",
       "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=",
       "dev": true,
       "requires": {
@@ -7284,7 +7298,7 @@
       "dependencies": {
         "ansi-regex": {
           "version": "1.1.1",
-          "resolved": "http://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz",
           "integrity": "sha1-QchHGUZGN15qGl0Qw8oFTvn8mA0=",
           "dev": true
         },
@@ -7296,7 +7310,7 @@
         },
         "strip-ansi": {
           "version": "2.0.1",
-          "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-2.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-2.0.1.tgz",
           "integrity": "sha1-32LBqpTtLxFOHQ8h/R1QSCt5pg4=",
           "dev": true,
           "requires": {
@@ -7357,7 +7371,7 @@
     },
     "wrap-ansi": {
       "version": "2.1.0",
-      "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
       "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
       "dev": true,
       "requires": {
@@ -7376,7 +7390,7 @@
         },
         "string-width": {
           "version": "1.0.2",
-          "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
           "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
           "dev": true,
           "requires": {
@@ -7461,7 +7475,7 @@
           "dependencies": {
             "string-width": {
               "version": "1.0.2",
-              "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+              "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
               "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
               "dev": true,
               "requires": {

+ 73 - 75
reader/views.py

@@ -2,6 +2,8 @@
 
 # noinspection PyUnresolvedReferences
 from datetime import datetime, timedelta, date
+from elasticsearch_dsl import Search
+from elasticsearch import Elasticsearch
 from sets import Set
 from random import choice
 from pprint import pprint
@@ -45,7 +47,7 @@ from sefaria.system.exceptions import InputError, PartialRefInputError, BookName
 # noinspection PyUnresolvedReferences
 from sefaria.client.util import jsonResponse
 from sefaria.history import text_history, get_maximal_collapsed_activity, top_contributors, make_leaderboard, make_leaderboard_condition, text_at_revision, record_version_deletion, record_index_deletion
-from sefaria.system.decorators import catch_error_as_json
+from sefaria.system.decorators import catch_error_as_json, json_response_decorator
 from sefaria.summaries import get_or_make_summary_node
 from sefaria.sheets import get_sheets_for_ref, public_sheets, get_sheets_by_tag, user_sheets, user_tags, recent_public_tags, sheet_to_dict, get_top_sheets, public_tag_list, group_sheets, get_sheet_for_panel, annotate_user_links
 from sefaria.utils.util import list_depth, text_preview
@@ -55,9 +57,10 @@ from sefaria.datatype.jagged_array import JaggedArray
 from sefaria.utils.calendars import get_all_calendar_items, get_keyed_calendar_items, this_weeks_parasha
 from sefaria.utils.util import short_to_long_lang_code, titlecase
 import sefaria.tracker as tracker
-from sefaria.system.cache import django_cache_decorator
-from sefaria.settings import USE_VARNISH, USE_NODE, NODE_HOST, DOMAIN_LANGUAGES, MULTISERVER_ENABLED
+from sefaria.system.cache import django_cache
+from sefaria.settings import USE_VARNISH, USE_NODE, NODE_HOST, DOMAIN_LANGUAGES, MULTISERVER_ENABLED, SEARCH_ADMIN
 from sefaria.system.multiserver.coordinator import server_coordinator
+from sefaria.helper.search import get_query_obj
 from django.utils.html import strip_tags
 
 if USE_VARNISH:
@@ -1460,28 +1463,32 @@ def index_api(request, title, v2=False, raw=False):
 
 
 @catch_error_as_json
+@json_response_decorator
+@django_cache(default_on_miss = True)
 def bare_link_api(request, book, cat):
-
     if request.method == "GET":
-        resp = jsonResponse(get_book_link_collection(book, cat), callback=request.GET.get("callback", None))
-        resp['Content-Type'] = "application/json; charset=utf-8"
+        resp = get_book_link_collection(book, cat)
         return resp
 
     elif request.method == "POST":
-        return jsonResponse({"error": "Not implemented."})
+        return {"error": "Not implemented."}
 
 
 @catch_error_as_json
+@json_response_decorator
+@django_cache(default_on_miss = True)
 def link_count_api(request, cat1, cat2):
     """
     Return a count document with the number of links between every text in cat1 and every text in cat2
     """
     if request.method == "GET":
-        resp = jsonResponse(get_link_counts(cat1, cat2))
+        resp = get_link_counts(cat1, cat2)
         return resp
 
     elif request.method == "POST":
-        return jsonResponse({"error": "Not implemented."})
+        return {"error": "Not implemented."}
+
+
 
 
 @catch_error_as_json
@@ -1905,43 +1912,43 @@ def version_status_api(request):
 
 simplified_toc = {}
 
+@json_response_decorator
+@django_cache(default_on_miss = True)
 def version_status_tree_api(request, lang=None):
-    global simplified_toc
-    key = lang or "none"
-    if not simplified_toc.get(key):
-        def simplify_toc(toc_node, path):
-            simple_nodes = []
-            for x in toc_node:
-                node_name = x.get("category", None) or x.get("title", None)
-                node_path = path + [node_name]
-                simple_node = {
-                    "name": node_name,
-                    "path": node_path
-                }
-                if "category" in x:
-                    if "contents" not in x:
-                        continue
-                    simple_node["type"] = "category"
-                    simple_node["children"] = simplify_toc(x["contents"], node_path)
-                elif "title" in x:
-                    query = {"title": x["title"]}
-                    if lang:
-                        query["language"] = lang
-                    simple_node["type"] = "index"
-                    simple_node["children"] = [{
-                           "name": u"{} ({})".format(v.versionTitle, v.language),
-                           "path": node_path + [u"{} ({})".format(v.versionTitle, v.language)],
-                           "size": v.word_count(),
-                           "type": "version"
-                       } for v in VersionSet(query)]
-                simple_nodes.append(simple_node)
-            return simple_nodes
-        simplified_toc[key] = simplify_toc(library.get_toc(), [])
-    return jsonResponse({
+    def simplify_toc(toc_node, path):
+        simple_nodes = []
+        for x in toc_node:
+            node_name = x.get("category", None) or x.get("title", None)
+            node_path = path + [node_name]
+            simple_node = {
+                "name": node_name,
+                "path": node_path
+            }
+            if "category" in x:
+                if "contents" not in x:
+                    continue
+                simple_node["type"] = "category"
+                simple_node["children"] = simplify_toc(x["contents"], node_path)
+            elif "title" in x:
+                query = {"title": x["title"]}
+                if lang:
+                    query["language"] = lang
+                simple_node["type"] = "index"
+                simple_node["children"] = [{
+                    "name": u"{} ({})".format(v.versionTitle, v.language),
+                    "path": node_path + [u"{} ({})".format(v.versionTitle, v.language)],
+                    "size": v.word_count(),
+                    "type": "version"
+                } for v in VersionSet(query)]
+            simple_nodes.append(simple_node)
+        return simple_nodes
+
+    result = simplify_toc(library.get_toc(), [])
+    return {
         "name": "Whole Library" + " ({})".format(lang) if lang else "",
         "path": [],
-        "children": simplified_toc[key]
-    }, callback=request.GET.get("callback", None))
+        "children": result
+    }
 
 
 def visualize_library(request, lang=None, cats=None):
@@ -2341,7 +2348,12 @@ def dictionary_completion_api(request, word, lexicon=None):
     # Number of results to return.  0 indicates no limit
     LIMIT = int(request.GET.get("limit", 10))
 
-    result = library.lexicon_auto_completer(lexicon).items(word)[:LIMIT]
+    if lexicon is None:
+        ac = library.cross_lexicon_auto_completer()
+        rs = ac.complete(word, LIMIT)
+        result = [[r, ac.title_trie[ac.normalizer(r)]["key"]] for r in rs]
+    else:
+        result = library.lexicon_auto_completer(lexicon).items(word)[:LIMIT]
     return jsonResponse(result)
 
 
@@ -3631,37 +3643,23 @@ def dummy_search_api(request):
     resp['Content-Type'] = "application/json; charset=utf-8"
     return resp
 
-# def search_api(request):
-#     # dict to define request parameters and their default values. None means parameter is required
-#     params = {
-#         "query": None,
-#         "size": 10,
-#         "from": 0,
-#         "type": None,  #
-#         "get_filters": False,
-#         "applied_filters": [],
-#         "field": None,
-#         "sort_type": None,
-#         "exact": False
-#     }
-#     param_vals = {}
-#     for p in params:
-#         param_vals[p] = request.GET.get(p, )
-#     query = request.GET.get("q")
-#     """
-#              query: query string
-#              size: size of result set
-#              from: from what result to start
-#              type: "sheet" or "text"
-#              get_filters: if to fetch initial filters
-#              applied_filters: filter query by these filters
-#              field: field to query in elastic_search
-#              sort_type: chonological or relevance
-#              exact: if query is exact
-#              success: callback on success
-#              error: callback on error
-#     """
-#     size = request.GET.get("size")
+
+@csrf_exempt
+def search_wrapper_api(request):
+    if request.method == "POST":
+        if "json" in request.POST:
+            j = request.POST.get("json")  # using form-urlencoded
+        else:
+            j = request.body  # using content-type: application/json
+        j = json.loads(j)
+        es_client = Elasticsearch(SEARCH_ADMIN)
+        search_obj = Search(using=es_client, index=j.get("type")).params(request_timeout=5)
+        search_obj = get_query_obj(search_obj=search_obj, **{k: v for k, v in j.items()})
+        response = search_obj.execute()
+        if response.success():
+            return jsonResponse(response.to_dict(), callback=request.GET.get("callback", None))
+        return jsonResponse({"error": "Error with connection to Elasticsearch. Total shards: {}, Shards successful: {}, Timed out: {}".format(response._shards.total, response._shards.successful, response.timed_out)}, callback=request.GET.get("callback", None))
+    return jsonResponse({"error": "Unsupported HTTP method."}, callback=request.GET.get("callback", None))
 
 
 @ensure_csrf_cookie

+ 2 - 1
requirements.txt

@@ -9,6 +9,7 @@ BeautifulSoup
 bleach==1.4.2
 rauth
 elasticsearch
+elasticsearch_dsl
 regex
 pytest
 mailchimp
@@ -37,4 +38,4 @@ user-agents
 django-user-agents
 psycopg2
 django-easy-timezones
-
+undecorated

+ 0 - 0
scripts/invalid refs.txt


+ 5 - 1
scripts/metrics.py

@@ -2,6 +2,7 @@
 import sys
 import os
 import datetime
+from pymongo.errors import DuplicateKeyError
 import django
 django.setup()
 
@@ -35,4 +36,7 @@ metrics = {
     "sheets": sheets,
 }
 
-db.metrics.save(metrics)
+try:
+    db.metrics.save(metrics)
+except DuplicateKeyError:
+    pass

+ 63 - 0
scripts/mishnah_yomit.py

@@ -0,0 +1,63 @@
+#encoding=utf-8
+import django
+django.setup()
+from sefaria.model import *
+
+#create new terms and categories
+t = Term()
+t.name = u"English Explanation of Mishnah"
+t.add_primary_titles(t.name, u"ביאור אנגלי על המשנה")
+t.save()
+
+c = Category()
+c.path = ["Modern Works", "English Explanation of Mishnah"]
+c.add_shared_term("English Explanation of Mishnah")
+c.save()
+mishnayot = ["Seder Moed", "Seder Kodashim", "Seder Zeraim", "Seder Nashim", "Seder Nezikin", "Seder Tahorot"]
+for mishnah in mishnayot:
+    c = Category()
+    c.path = ["Modern Works", "English Explanation of Mishnah", mishnah]
+    c.add_shared_term(mishnah)
+    c.save()
+
+
+#change index titles and version titles
+pre_new_he = u"ביאור אנגלי על "
+pre_new_en = "English Explanation of "
+indices = library.get_indices_by_collective_title("Mishnah Yomit")
+for i, index in enumerate(indices):
+    print "CHANGING INDEX TITLE for {}".format(index)
+    print index
+    index = library.get_index(index)
+    mishnah_en = index.base_text_titles[0]
+    mishnah_he = library.get_index(mishnah_en).get_title('he')
+    new_en = pre_new_en + mishnah_en
+    new_he = pre_new_he + mishnah_he
+    index.set_title(new_he, "he")
+    index.save()
+    index.set_title(new_en, "en")
+    index.save()
+    index.collective_title = "English Explanation of Mishnah"
+    new_cat = list(index.categories)
+    new_cat[1] = "English Explanation of Mishnah"
+    index.categories = new_cat
+    index.save()
+    print "NOW CHANGING VERSION STATE TITLE"
+    index = library.get_index(new_en)
+    vs = index.versionSet()
+    v = vs[0]
+    v.versionTitle = "Mishnah Yomit by Dr. Joshua Kulp"
+    v.save()
+
+#delete old categories
+library.rebuild(include_toc=True)
+from sefaria.model.category import TocCategory
+c = Category().load({"path": ["Modern Works", "Mishnah Yomit"]})
+for toc_obj in c.get_toc_object().all_children():
+    if isinstance(toc_obj, TocCategory):
+        print toc_obj, toc_obj.get_category_object().can_delete()
+        toc_obj.get_category_object().delete()
+
+library.rebuild(include_toc=True)
+c = Category().load({"path": ["Modern Works", "Mishnah Yomit"]})
+c.delete()

+ 45 - 0
scripts/netzach_yisrael.py

@@ -0,0 +1,45 @@
+#encoding=utf-8
+import django
+django.setup()
+from sefaria.helper.schema import *
+
+root = SchemaNode()
+root.add_primary_titles("Netzach Yisrael", u"נצח ישראל")
+intro = JaggedArrayNode()
+intro.add_structure(["Paragraph"])
+intro.add_shared_term("Introduction")
+intro.key = "intro"
+intro.validate()
+default = JaggedArrayNode()
+default.default = True
+default.key = "default"
+default.add_structure(["Chapter", "Paragraph"])
+default.validate()
+root.append(intro)
+root.append(default)
+root.validate()
+schema = root.serialize()
+
+mapping = {}
+for i in range(1, 65):
+    actual_value = i-1
+    if actual_value == 0:
+        mapping["Netzach Yisrael 1"] = "Netzach Yisrael, Introduction"
+    else:
+        mapping["Netzach Yisrael {}".format(i)] = "Netzach Yisrael {}".format(actual_value)
+
+
+migrate_to_complex_structure("Netzach Yisrael", schema, mapping)
+
+
+library.rebuild(include_toc=True)
+
+i = library.get_index("Netzach Yisrael")
+section_refs = i.all_section_refs()
+for ref in section_refs:
+    new_text = ref.text('he').text[1:]
+    tc = TextChunk(ref, vtitle="Netzach Yisrael", lang="he")
+    tc.text = new_text
+    tc.save(force_save=True)
+
+

+ 42 - 0
scripts/regenerate_explorer_data.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+import argparse
+import django
+django.setup()
+from sefaria.system.database import db
+from sefaria.model import *
+from sefaria.system.cache import django_cache
+from sefaria.model.link import get_book_link_collection, get_link_counts
+
+
+
+def redo_bare_links_api(cat1, cat2):
+    cat1idxs = library.get_indexes_in_category(cat1)
+    cat2idxs = library.get_indexes_in_category(cat2)
+    for c1idx in cat1idxs:
+        print "bare_link_api:, Book: {}, Category: {}".format(c1idx, cat2)
+        django_cache(action="set", cache_prefix='bare_link_api')(get_book_link_collection)(book=c1idx, cat=cat2)
+    for c2idx in cat2idxs:
+        print "bare_link_api:, Book: {}, Category: {}".format(c2idx, cat1)
+        django_cache(action="set", cache_prefix='bare_link_api')(get_book_link_collection)(book=c2idx, cat=cat1)
+
+
+def redo_link_count_api(cat1, cat2):
+    print "link_count_api:, Category1: {}, Category2: {}".format(cat1, cat2)
+    django_cache(action="set", cache_prefix='link_count_api')(get_link_counts)(cat1=cat1, cat2=cat2)
+
+
+
+
+
+
+
+
+
+""" The main function, runs when called from the CLI"""
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cat1", help="first category of text to calculate bare links")
+    parser.add_argument("--cat2", help="second category of text to calculate bare links")
+    args = parser.parse_args()
+    redo_bare_links_api(args.cat1, args.cat2)
+    redo_link_count_api(args.cat1, args.cat2)

+ 1 - 1
scripts/rewrite_rashis_in_tanchuma.py

@@ -154,4 +154,4 @@ if __name__ == "__main__":
     execute()
 
 
-
+    

+ 20 - 0
scripts/sheet_tag_usage_count.py

@@ -0,0 +1,20 @@
+import django
+django.setup()
+
+import csv
+from sefaria.model import *
+from sefaria.system.database import db
+
+
+
+sheet_tag_usage_count = [["tag","count"]]
+
+tags = db.sheets.distinct("tags")
+for tag in tags:
+    if tag: #required b/c apparently `None` is a tag somehow...
+        sheets_with_tag_count = db.sheets.find({"tags": tag}).count()
+        sheet_tag_usage_count.append([tag.encode("utf-8"),sheets_with_tag_count])
+
+with open("output.csv", "wb") as f:
+	writer = csv.writer(f)
+	writer.writerows(sheet_tag_usage_count)

+ 47 - 0
scripts/sheets_with_non_library_content.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+import django
+django.setup()
+
+from sefaria.model import *
+from sefaria.system.database import db
+
+import csv
+
+sheets = db.sheets.find({"status": "public"})
+
+
+sheets_with_non_library_content = [["sheet_id","sheet_title","like_count"]]
+
+for sheet in sheets:
+	row = []
+	sources = sheet.get("sources", [])
+	for source in sources:
+		if "comment" in source:
+			row.append(sheet.get("id", ""))
+			row.append(sheet.get("title", "").encode("utf-8"))
+			row.append(len(sheet.get("likes", [])))
+			break
+		elif "outsideBiText" in source:
+			row.append(sheet.get("id", ""))
+			row.append(sheet.get("title", "").encode("utf-8"))
+			row.append(len(sheet.get("likes", [])))
+			break
+		elif "outsideText" in source:
+			row.append(sheet.get("id", ""))
+			row.append(sheet.get("title", "").encode("utf-8"))
+			row.append(len(sheet.get("likes", [])))
+			break
+		elif "media" in source:
+			row.append(sheet.get("id", ""))
+			row.append(sheet.get("title", "").encode("utf-8"))
+			row.append(len(sheet.get("likes", [])))
+			break
+
+	if len(row) > 0:
+		sheets_with_non_library_content.append(row)
+		print row
+
+with open("output.csv", "wb") as f:
+	writer = csv.writer(f)
+	writer.writerows(sheets_with_non_library_content)

+ 53 - 0
scripts/top_pagerank_parsha.py

@@ -0,0 +1,53 @@
+import codecs
+import json
+import re
+import unicodecsv
+from collections import defaultdict
+import heapq
+import math as mathy
+import django
+django.setup()
+
+from sefaria.model import *
+from sefaria.system.exceptions import InputError
+
+
+def argmax(iterable, n=1):
+    if n == 1:
+        return max(enumerate(iterable), key=lambda x: x[1])[0]
+    else:
+        return heapq.nlargest(n, xrange(len(iterable)), iterable.__getitem__)
+
+ref2parsha = {}
+parsha2pr = defaultdict(list)
+for i in library.get_indexes_in_category("Torah", full_records=True):
+    parshiot = i.alt_structs["Parasha"]["nodes"]
+    for p in parshiot:
+        r = Ref(p["wholeRef"])
+        for pasuk in r.range_list():
+            ref2parsha[pasuk.normal()] = p["sharedTitle"]
+
+my_csv = []
+with codecs.open("../static/sheetrank.json", "rb", encoding="utf8") as fin:
+    sheetrank = json.load(fin, encoding="utf8")
+with codecs.open("../static/pagerank.json", "rb", encoding="utf8") as fin:
+    jin = json.load(fin, encoding="utf8")
+    for ii, (tref, pr) in enumerate(jin):
+        if ii % 1000 == 0:
+            print ii
+        try:
+            oref = Ref(tref)
+            if len(oref.text("he").text) == 0:
+                continue
+            new_pr = mathy.log(pr) + 20
+            sheet_pr = (1.0 + sheetrank[tref]["count"] / 5)**2 if tref in sheetrank else (1.0 / 5) ** 2
+            my_csv += [{"Ref": tref, "PR": new_pr * sheet_pr}]
+        except InputError:
+            continue
+
+
+with open("../static/all_pagerank.csv", "wb") as fout:
+    my_csv.sort(key=lambda x: Ref(x["Ref"]).order_id())
+    c = unicodecsv.DictWriter(fout, ["Ref", "PR"])
+    c.writeheader()
+    c.writerows(my_csv)

+ 5 - 1
sefaria/client/util.py

@@ -3,8 +3,9 @@ import json
 from rauth import OAuth2Service
 from datetime import datetime
 
-from django.http import HttpResponse
+from django.http import HttpResponse, JsonResponse
 from django.core.mail import EmailMultiAlternatives
+from functools import wraps
 
 from sefaria import local_settings as sls
 
@@ -18,6 +19,9 @@ def jsonResponse(data, callback=None, status=200):
     except AttributeError:
         pass
 
+    if data is None:
+        data = {"error": 'No data available'}
+
     if "_id" in data:
         data["_id"] = str(data["_id"])
 

+ 3 - 1
sefaria/helper/schema.py

@@ -510,8 +510,10 @@ def change_node_structure(ja_node, section_names, address_types=None, upsize_in_
             ref_name = ref_name.replace(index.title, v.get_index().title)
             chunk = TextChunk(Ref(ref_name), lang=v.language, vtitle=v.versionTitle)
         ja = chunk.ja()
+        if ja.get_depth() == 0:
+            continue
 
-        if upsize_in_place or ja.get_depth() == 0:
+        if upsize_in_place:
             wrapper = chunk.text
             for i in range(delta):
                 wrapper = [wrapper]

+ 145 - 0
sefaria/helper/search.py

@@ -0,0 +1,145 @@
+from functools import wraps
+from elasticsearch_dsl import Q, Search
+from elasticsearch_dsl.query import Bool, Regexp, Term
+import re
+
+
+def default_list(param):
+    if param is None:
+        return []
+    return param
+
+
+def default_bool(param):
+    if param is None:
+        return False
+    return param
+
+
+def default_search(param):
+    if param is None:
+        return Search()
+    return param
+
+
+def param_fixer(func):
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        func_params = func.func_code.co_varnames[:func.func_code.co_argcount]
+        extra_params = set(kwargs.keys()) - set(func_params)
+        for extra in extra_params:
+            kwargs.pop(extra)
+        args = list(args)
+        params_with_defaults = {
+            u"source_proj": default_bool,
+            u"filters": default_list,
+            u"filter_fields": default_list,
+            u"aggs": default_list,
+            u"sort_fields": default_list,
+            u"sort_reverse": default_bool,
+            u"search_obj": default_search
+        }
+        for param, setter in params_with_defaults.items():
+            i = func_params.index(param)
+            if len(args) > i:
+                # in args
+                args[i] = setter(args[i])
+            else:
+                # maybe in kwargs
+                kwargs[param] = setter(kwargs.get(param, None))
+        return func(*args, **kwargs)
+    return wrapper
+
+
+@param_fixer
+def get_query_obj(
+        query,
+        type=u"text",
+        field=u"exact",
+        source_proj=False,
+        slop=0,
+        start=0,
+        size=100,
+        filters=None,
+        filter_fields=None,
+        aggs=None,
+        sort_method=u"sort",
+        sort_fields=None,
+        sort_reverse=False,
+        sort_score_missing=0,
+        search_obj=None):
+    """
+
+    :param query :str:
+    :param type :str: one_of("text", "sheet")
+    :param field :str: which field do you want to query? usually either "exact", "naive_lemmatizer" or "content"
+    :param source_proj :str or list(str) or bool: if False, don't return _source. o/w only return fields specified
+    :param slop :int: max distance allowed b/w words in the query. 0 is an exact match
+    :param start :int: pagination start
+    :param size :int: page size
+    :param filters :list(str): list of filters you've applied
+    :param filter_fields :list(str): list of fields each filter is filtering on. must be same size as `filters` usually "path", "group" or "tags"
+    :param aggs :list(str): list of fields to aggregate on. usually "path", "group" or "tags"
+    :param sort_method :str: how to sort. either "sort" or "score"
+    :param sort_fields :list(str): which fields to sort on. sorts are applied in order stably
+    :param sort_reverse :bool: should the sorting be reversed?
+    :param sort_score_missing :float: in the case of `sort_method = "score"` what value to use if `sort_fields` doesn't exist on a doc
+    :param search_obj :Search: object to add the query, sorting, filters etc. optional
+    :return: Search object with all the stuff ready to execute
+    """
+    search_obj = search_obj.source(source_proj)
+    query = re.sub(ur"(\S)\"(\S)", ur"\1\u05f4\2", query)  # Replace internal quotes with gershaim.
+    core_query = Q(u"match_phrase", **{field: {u"query": query, u"slop": slop}})
+
+    # sort
+    if sort_method == u"sort":
+        search_obj = search_obj.sort(*[u"{}{}".format(u"-" if sort_reverse else u"", f) for f in sort_fields])
+
+    # aggregations
+    if len(aggs) > 0:
+        for a in aggs:
+            search_obj.aggs.bucket(a, u"terms", field=a, size=10000)
+
+    # filters
+    if len(filters) == 0:
+        inner_query = core_query
+    else:
+        inner_query = Bool(must=core_query, filter=get_filter_obj(type, filters, filter_fields))
+
+    # finish up
+    if sort_method == u"score" and len(sort_fields) == 1:
+        search_obj.query = {
+            u"function_score": {
+                u"query": inner_query.to_dict(),
+                u"field_value_factor": {
+                    u"field": sort_fields[0],
+                    u"missing": sort_score_missing
+                }
+            }
+        }
+    else:
+        search_obj.query = inner_query
+    search_obj = search_obj.highlight(field, fragment_size=200, pre_tags=[u"<b>"], post_tags=[u"</b>"])
+    return search_obj[start:start + size]
+
+
+def get_filter_obj(type, filters, filter_fields):
+    if len(filter_fields) == 0:
+        filter_fields = [None] * len(filters)  # use default filter_field for query type (defined in make_filter())
+    unique_fields = set(filter_fields)
+    must_bools = []
+    for agg_type in unique_fields:
+        type_filters = filter(lambda x: x[1] == agg_type, zip(filters, filter_fields))
+        should_bool = Bool(should=[make_filter(type, agg_type, f) for f, t in type_filters])
+        must_bools += [should_bool]
+    return Bool(must=must_bools)
+
+
+def make_filter(type, agg_type, agg_key):
+    if type == u"text":
+        # filters with '/' might be leading to books. also, very unlikely they'll match an false positives
+        reg = re.escape(agg_key) + (u".*" if u"/" in agg_key else u"/.*")
+        return Regexp(path=reg)
+    elif type == u"sheet":
+        return Term(**{agg_type: agg_key})

+ 14 - 1
sefaria/helper/tests/schema_test.py

@@ -1,4 +1,6 @@
 # encoding=utf-8
+import django
+django.setup()
 from sefaria.model import *
 from sefaria.helper import schema
 from sefaria.system.exceptions import BookNameError
@@ -68,6 +70,15 @@ def setup_module():
         "chapter": root.create_skeleton()
     }).save()
 
+    # an empty version
+    v = Version({
+        "language": "en",
+        "title": "Delete Me",
+        "versionSource": "http://foobar.com",
+        "versionTitle": "Schema Test Blank",
+        "chapter": root.create_skeleton()
+    }).save()
+
     p1 = [['Part1 part1', 'Part1'], ['Part1'], ['Part1', '', 'part1']]
     chunk = TextChunk(Ref('Delete Me, Part1'), 'en', 'Schema Test')
     chunk.text = p1
@@ -121,7 +132,7 @@ def teardown_module():
     ls.delete()
     ns = NoteSet({"ref": {"$regex": "Delete Me.*"}})
     ns.delete()
-    v = Version().load({'title': 'Delete Me'})
+    v = VersionSet({'title': 'Delete Me'})
     v.delete()
     i = Index().load({'title': 'Delete Me'})
     i.delete()
@@ -314,6 +325,8 @@ def test_change_node_structure():
     assert node.depth == 3
     chunk = TextChunk(Ref('Delete Me, Part1'), 'en', 'Schema Test')
     assert chunk.text == [[['Part1 part1'], ['Part1']], [['Part1']], [['Part1'], [], ['part1']]]
+    blank_chunk = TextChunk(Ref('Delete Me, Part1'), 'en', 'Schema Test Blank')
+    assert len(blank_chunk.text) == 0
     assert isinstance(Link().load({'refs': ['Delete Me, Part1 1:1:1', 'Shabbat 2a:5'],}), Link)
     assert isinstance(Link().load({'refs': ['Delete Me, Part1 2:1:1', 'Delete Me, Part2 2:1'], }), Link)
     assert isinstance(Link().load({'refs': ['Delete Me, Part1 3:1', 'Shabbat 2a:5'], }), Link)

File diff suppressed because it is too large
+ 40 - 0
sefaria/helper/tests/search_test.py


+ 44 - 14
sefaria/model/autospell.py

@@ -38,7 +38,8 @@ class AutoCompleter(object):
     An AutoCompleter object provides completion services - it is the object in this module designed to be used by the Library.
     It instantiates objects that provide string completion according to different algorithms.
     """
-    def __init__(self, lang, lib, include_people=False, include_categories=False, include_parasha=False, *args, **kwargs):
+    def __init__(self, lang, lib, include_titles=True, include_people=False, include_categories=False,
+                 include_parasha=False, include_lexicons=False, *args, **kwargs):
         """
 
         :param lang:
@@ -55,16 +56,18 @@ class AutoCompleter(object):
         self.title_trie = TitleTrie(lang, *args, **kwargs)
         self.spell_checker = SpellChecker(lang)
         self.ngram_matcher = NGramMatcher(lang)
+        self.other_lang_ac = None
+        self.prefer_longest = True  # True for titles, False for dictionary entries.  AC w/ combo of two may be tricky.
 
         # Titles in library
-        title_node_dict = self.library.get_title_node_dict(lang)
-        tnd_items = title_node_dict.items()
-        titles = [t for t, d in tnd_items]
-        normal_titles = [self.normalizer(t) for t, d in tnd_items]
-        self.title_trie.add_titles_from_title_node_dict(tnd_items, normal_titles)
-        self.spell_checker.train_phrases(normal_titles)
-        self.ngram_matcher.train_phrases(titles, normal_titles)
-
+        if include_titles:
+            title_node_dict = self.library.get_title_node_dict(lang)
+            tnd_items = title_node_dict.items()
+            titles = [t for t, d in tnd_items]
+            normal_titles = [self.normalizer(t) for t, d in tnd_items]
+            self.title_trie.add_titles_from_title_node_dict(tnd_items, normal_titles)
+            self.spell_checker.train_phrases(normal_titles)
+            self.ngram_matcher.train_phrases(titles, normal_titles)
         if include_categories:
             categories = self._get_main_categories(library.get_toc_tree().get_root())
             category_names = [c.primary_title(lang) for c in categories]
@@ -87,7 +90,34 @@ class AutoCompleter(object):
             self.title_trie.add_titles_from_set(ps, "all_names", "primary_name", "key")
             self.spell_checker.train_phrases(person_names)
             self.ngram_matcher.train_phrases(person_names, normal_person_names)
+        if include_lexicons:
+            # languages get muddy for lexicons
+            self.prefer_longest = False
+            wfs = WordFormSet({"generated_by": {"$ne": "replace_shorthand"}})
+
+            for wf in wfs:
+                self.title_trie[self.normalizer(wf.form)] = {
+                    "title": wf.form,
+                    "key": wf.form,
+                    "type": "word_form",
+                    "is_primary": True
+                }
+                if not hasattr(wf, "c_form"):
+                    continue
+                self.title_trie[self.normalizer(wf.c_form)] = {
+                    "title": wf.c_form,
+                    "key": wf.form,
+                    "type": "word_form",
+                    "is_primary": True
+                }
+
+            forms = [getattr(wf, "c_form", wf.form) for wf in wfs]
+            normal_forms = [self.normalizer(wf) for wf in forms]
+            self.spell_checker.train_phrases(forms)
+            self.ngram_matcher.train_phrases(forms, normal_forms)
 
+    def set_other_lang_ac(self, ac):
+        self.other_lang_ac = ac
 
     @staticmethod
     def _get_main_categories(otoc):
@@ -128,10 +158,9 @@ class AutoCompleter(object):
             return completions
 
         # No results. Try letter swap
-        if not redirected:
-            other_language = "he" if self.lang == "en" else "en"
+        if not redirected and self.other_lang_ac:
             swapped_string = hebrew.swap_keyboards_for_string(instring)
-            return self.library.full_auto_completer(other_language).complete(swapped_string, limit, redirected=True)
+            return self.other_lang_ac.complete(swapped_string, limit, redirected=True)
 
         return []
 
@@ -217,7 +246,8 @@ class Completions(object):
         """
 
         try:
-            all_continuations = self.auto_completer.title_trie.items(str)[::-1]
+            skip = -1 if self.auto_completer.prefer_longest else 1
+            all_continuations = self.auto_completer.title_trie.items(str)[::skip]
         except KeyError:
             return []
 
@@ -227,7 +257,7 @@ class Completions(object):
         non_primary_matches = []
         for k, v in all_continuations:
             if v["is_primary"] and v["key"] not in self.keys_covered:
-                if v["type"] == "ref":
+                if v["type"] == "ref" or v["type"] == "word_form":
                     self.completions += [v["title"]]
                 else:
                     self.completions.insert(0, v["title"])

+ 1 - 9
sefaria/model/link.py

@@ -269,18 +269,11 @@ def process_index_delete_in_links(indx, **kwargs):
 
 #get_link_counts() and get_book_link_collection() are used in Link Explorer.
 #They have some client formatting code in them; it may make sense to move them up to sefaria.client or sefaria.helper
-link_counts = {}
 def get_link_counts(cat1, cat2):
     """
     Returns a list of book to book link counts for books within `cat1` and `cat2`
     Parameters may name either a category or a individual book
     """
-
-    global link_counts
-    key = cat1 + "-" + cat2
-    if link_counts.get(key):
-        return link_counts[key]
-
     titles = []
     for c in [cat1, cat2]:
         ts = text.library.get_indexes_in_category(c)
@@ -297,11 +290,10 @@ def get_link_counts(cat1, cat2):
         for title2 in titles[1]:
             re1 = r"^{} \d".format(title1)
             re2 = r"^{} \d".format(title2)
-            links = LinkSet({"$and": [{"refs": {"$regex": re1}}, {"refs": {"$regex": re2}}]})  # db.links.find({"$and": [{"refs": {"$regex": re1}}, {"refs": {"$regex": re2}}]})
+            links = LinkSet({"$and": [{"refs": {"$regex": re1}}, {"refs": {"$regex": re2}}]})
             if links.count():
                 result.append({"book1": title1, "book2": title2, "count": links.count()})
 
-    link_counts[key] = result
     return result
 
 

+ 3 - 17
sefaria/model/notification.py

@@ -69,9 +69,9 @@ class GlobalNotification(abst.AbstractMongoRecord):
             i = self.content.get("index")
             v = self.content.get("version")
             l = self.content.get("language")
-            assert i
-            assert v
-            assert l
+
+            assert i and v and l
+
             version = Version().load({
                 "title": i,
                 "versionTitle": v,
@@ -120,13 +120,6 @@ class GlobalNotification(abst.AbstractMongoRecord):
         self.content["en"] = msg
         return self
 
-    """
-    def to_HTML(self):
-        html = render_to_string("elements/notification.html", {"notification": self}).strip()
-        html = re.sub("\n", "", html)
-        return html
-    """
-
     def contents(self):
         d = super(GlobalNotification, self).contents()
         d["_id"] = self.id
@@ -134,7 +127,6 @@ class GlobalNotification(abst.AbstractMongoRecord):
 
         return d
 
-
     @property
     def id(self):
         return str(self._id)
@@ -153,12 +145,6 @@ class GlobalNotificationSet(abst.AbstractMongoSet):
     def contents(self):
         return [n.contents() for n in self]
 
-    """
-    def to_HTML(self):
-        html = [n.to_HTML() for n in self]
-        return "".join(html)
-    """
-
 
 class Notification(abst.AbstractMongoRecord):
     collection   = 'notifications'

+ 22 - 1
sefaria/model/text.py

@@ -3469,7 +3469,7 @@ class Ref(object):
             if as_list:
                 return [u"{}{}".format(escaped_book, p) for p in patterns]
             else:
-                return "u%s(%s)" % (escaped_book, u"|".join(patterns))
+                return u"%s(%s)" % (escaped_book, u"|".join(patterns))
 
     def ref_regex_query(self):
         """
@@ -4054,6 +4054,7 @@ class Library(object):
         self._full_auto_completer = {}
         self._ref_auto_completer = {}
         self._lexicon_auto_completer = {}
+        self._cross_lexicon_auto_completer = None
 
         # Term Mapping
         self._simple_term_mapping = {}
@@ -4182,24 +4183,42 @@ class Library(object):
             lang: AutoCompleter(lang, library, include_people=True, include_categories=True, include_parasha=True) for lang in self.langs
         }
 
+        for lang in self.langs:
+            self._full_auto_completer[lang].set_other_lang_ac(self._full_auto_completer["he" if lang == "en" else "en"])
+
     def build_ref_auto_completer(self):
         from autospell import AutoCompleter
         self._ref_auto_completer = {
             lang: AutoCompleter(lang, library, include_people=False, include_categories=False, include_parasha=False) for lang in self.langs
         }
 
+        for lang in self.langs:
+            self._ref_auto_completer[lang].set_other_lang_ac(self._ref_auto_completer["he" if lang == "en" else "en"])
+
     def build_lexicon_auto_completers(self):
         from autospell import LexiconTrie
         self._lexicon_auto_completer = {
             lexicon: LexiconTrie(lexicon) for lexicon in ["Jastrow Dictionary", "Klein Dictionary"]
         }
 
+    def build_cross_lexicon_auto_completer(self):
+        from autospell import AutoCompleter
+        self._cross_lexicon_auto_completer = AutoCompleter("he", library, include_titles=False, include_lexicons=True)
+
+    def cross_lexicon_auto_completer(self):
+        if self._cross_lexicon_auto_completer is None:
+            logger.warning("Failed to load cross lexicon auto completer, rebuilding.")
+            self.build_cross_lexicon_auto_completer()  # I worry that these could pile up.
+            logger.warning("Built cross lexicon auto completer.")
+        return self._cross_lexicon_auto_completer
+
     def lexicon_auto_completer(self, lexicon):
         try:
             return self._lexicon_auto_completer[lexicon]
         except KeyError:
             logger.warning("Failed to load {} auto completer, rebuilding.".format(lexicon))
             self.build_lexicon_auto_completers()  # I worry that these could pile up.
+            logger.warning("Built {} auto completer.".format(lexicon))
             return self._lexicon_auto_completer[lexicon]
 
     def full_auto_completer(self, lang):
@@ -4208,6 +4227,7 @@ class Library(object):
         except KeyError:
             logger.warning("Failed to load full {} auto completer, rebuilding.".format(lang))
             self.build_full_auto_completer()  # I worry that these could pile up.
+            logger.warning("Built full {} auto completer.".format(lang))
             return self._full_auto_completer[lang]
 
     def ref_auto_completer(self, lang):
@@ -4216,6 +4236,7 @@ class Library(object):
         except KeyError:
             logger.warning("Failed to load {} ref auto completer, rebuilding.".format(lang))
             self.build_ref_auto_completer()  # I worry that these could pile up.
+            logger.warning("Built {} ref auto completer.".format(lang))
             return self._ref_auto_completer[lang]
 
     def recount_index_in_toc(self, indx):

+ 4 - 2
sefaria/model/user_profile.py

@@ -21,7 +21,6 @@ from sefaria.model.text import Ref
 from sefaria.system.database import db
 from sefaria.utils.util import epoch_time, concise_natural_time
 from django.utils import translation
-from django.contrib.humanize.templatetags.humanize import naturaltime
 
 
 class UserHistory(abst.AbstractMongoRecord):
@@ -95,7 +94,10 @@ class UserHistory(abst.AbstractMongoRecord):
             except KeyError:
                 pass
         if kwargs.get("natural_time", False):
-            d["natural_time"] = concise_natural_time(datetime.utcfromtimestamp(d["time_stamp"]))
+            d["natural_time"] = {
+                "en": concise_natural_time(datetime.utcfromtimestamp(d["time_stamp"]), lang="en"),
+                "he": concise_natural_time(datetime.utcfromtimestamp(d["time_stamp"]), lang="he")
+            }
         return d
 
 

+ 28 - 3
sefaria/pagesheetrank.py

@@ -6,6 +6,8 @@ import math
 import numpy
 import random
 import json
+import time
+from pymongo.errors import AutoReconnect
 from collections import defaultdict, OrderedDict
 from sefaria.model import *
 from sefaria.system.exceptions import InputError, NoVersionFoundError
@@ -252,6 +254,29 @@ def test_pagerank(a,b):
     ranked = pagerank(g, a, verbose=True, tolerance=b)
     print ranked
 
+
+def get_all_sheets(tries=0, page=0):
+    limit = 1000
+    has_more = True
+    while has_more:
+        try:
+            temp_sheets = list(db.sheets.find().skip(page*limit).limit(limit))
+        except AutoReconnect as e:
+            tries += 1
+            if tries >= 200:
+                print "Tried: {} times".format(tries)
+                raise e
+            time.sleep(5)
+            continue
+        has_more = False
+        for s in temp_sheets:
+            has_more = True
+            yield s
+        page += 1
+
+
+
+
 def calculate_sheetrank():
 
     def count_sources(sources):
@@ -289,12 +314,12 @@ def calculate_sheetrank():
         return temp_sources_count
 
     graph = defaultdict(int)
-    sheets = db.sheets.find()
-    total = sheets.count()
+    len_sheets = db.sheets.find().count()
+    sheets = get_all_sheets()
     sources_count = 0
     for i, sheet in enumerate(sheets):
         if i % 1000 == 0:
-            print "{}/{}".format(i, total)
+            print "{}/{}".format(i, len_sheets)
         if "sources" not in sheet:
             continue
         sources_count += count_sources(sheet["sources"])

+ 4 - 6
sefaria/search.py

@@ -540,20 +540,18 @@ class TextIndexer(object):
         # slower than `cls.index_version` but useful when you don't want the overhead of loading all versions into cache
         cls.merged = merged
         cls.index_name = index_name
-        cls.best_time_period = cls.curr_index.best_time_period()
         cls.curr_index = oref.index
+        cls.best_time_period = cls.curr_index.best_time_period()
         cls.trefs_seen = set()
         version_priority = 0
-        version = None
         if not merged:
             for priority, v in enumerate(cls.get_ref_version_list(oref)):
                 if v['versionTitle'] == version_title:
                     version_priority = priority
-                    version = v
         content = TextChunk(oref, lang, vtitle=version_title).ja().flatten_to_string()
         categories = cls.curr_index.categories
         tref = oref.normal()
-        doc = cls.make_text_index_document(tref, oref.he_normal(), version, lang, version_priority, content, categories)
+        doc = cls.make_text_index_document(tref, oref.he_normal(), version_title, lang, version_priority, content, categories)
         id = make_text_doc_id(tref, version_title, lang)
         es_client.index(index_name, "text", doc, id)
 
@@ -619,9 +617,9 @@ class TextIndexer(object):
         else:
             comp_start_date = 3000  # far in the future
 
-        section_ref = tref[:tref.rfind(u":")] if u":" in tref else (tref[:re.search(ur" \d+$", tref).start()] if re.search(ur" \d+$", tref) is not None else tref)
+        # section_ref = tref[:tref.rfind(u":")] if u":" in tref else (tref[:re.search(ur" \d+$", tref).start()] if re.search(ur" \d+$", tref) is not None else tref)
 
-        pagerank = math.log(pagerank_dict[section_ref]) + 20 if section_ref in pagerank_dict else 1.0
+        pagerank = math.log(pagerank_dict[tref]) + 20 if tref in pagerank_dict else 1.0
         sheetrank = (1.0 + sheetrank_dict[tref]["count"] / 5)**2 if tref in sheetrank_dict else (1.0 / 5) ** 2
         return {
             "ref": tref,

+ 2 - 2
sefaria/sheets.py

@@ -19,7 +19,7 @@ from sefaria.model.user_profile import UserProfile, annotate_user_list, public_u
 from sefaria.model.group import Group, GroupSet
 from sefaria.utils.util import strip_tags, string_overlap, titlecase
 from sefaria.system.exceptions import InputError
-from sefaria.system.cache import django_cache_decorator
+from sefaria.system.cache import django_cache
 from history import record_sheet_publication, delete_sheet_publication
 from settings import SEARCH_INDEX_ON_SAVE
 import search
@@ -601,7 +601,7 @@ def get_last_updated_time(sheet_id):
 	return sheet["dateModified"]
 
 
-@django_cache_decorator(time=(60 * 60))
+@django_cache(timeout=(60 * 60))
 def public_tag_list(sort_by="alpha"):
 	"""
 	Returns a list of all public tags, sorted either alphabetically ("alpha") or by popularity ("count")

+ 32 - 10
sefaria/system/cache.py

@@ -1,6 +1,11 @@
 
 import hashlib
 import sys
+from functools import wraps
+from django.http import HttpRequest
+
+import logging
+logger = logging.getLogger(__name__)
 
 try:
     from sefaria.settings import USE_VARNISH
@@ -37,30 +42,47 @@ def cache_get_key(*args, **kwargs):
     return key
 
 
-def django_cache_decorator(time=300, cache_key='', cache_type=None):
+def django_cache(action="get", timeout=None, cache_key='', cache_prefix = None, default_on_miss = False, default_on_miss_value=None, cache_type=None):
     """
     Easily add caching to a function in django
     """
-    cache_instance = get_cache_factory(cache_type)
     if not cache_key:
         cache_key = None
 
     def decorator(fn):
+        fn.func_dict["django_cache"] = True
+        @wraps(fn)
         def wrapper(*args, **kwargs):
             #logger.debug([args, kwargs])
 
             # Inner scope variables are read-only so we set a new var
             _cache_key = cache_key
+            do_actual_func = False
 
             if not _cache_key:
-                _cache_key = cache_get_key(fn.__name__, *args, **kwargs)
-
-            #logger.debug(['_cach_key.......',_cache_key])
-            result = cache_instance.get(_cache_key)
+                key_args = args[:]
+                if len(key_args) and isinstance(key_args[0], HttpRequest): # we dont want a HttpRequest to form part of the cache key, it wont be replicatable.
+                    key_args = key_args[1:]
+                _cache_key = cache_get_key(cache_prefix if cache_prefix else fn.__name__, *key_args, **kwargs)
+
+            if action in ["reset", "set"]:
+                do_actual_func = True
+                try:
+                    delete_cache_elem(_cache_key, cache_type=cache_type)
+                except:
+                    pass
+                result = None
+            else:
+                #logger.debug(['_cach_key.......',_cache_key])
+                result = get_cache_elem(_cache_key, cache_type=cache_type)