Merge pull request #43 from hatnote/sendy

Add sendy integration
hatnote · Sep 20, 2020 · af08ec7 · af08ec7
2 parents 0e002da + 276915f
commit af08ec7
Show file tree

Hide file tree

Showing 66 changed files with 1,978 additions and 943 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,12 +1,18 @@
-Werkzeug==0.9.4
-argparse==1.2.1
--e git+https://github.com/mahmoud/clastic.git#egg=clastic
+ashes==0.7.6
+Babel==1.3
+certifi==2020.6.20
+chardet==3.0.4
+-e git+https://github.com/mahmoud/clastic.git@d6afb4ae849e8b463885465a01e28c1708d12a47#egg=clastic
 docopt==0.4.0
+idna==2.10
 mailchimp==2.0.7
--e git+https://github.com/slaporte/wapiti.git#egg=wapiti
-wsgiref==0.1.2
-cronfed==0.2.1
-ashes==0.7.6
-PyYAML==3.11
 python-dateutil==2.4.2
-Babel==2.0
+pytz==2014.7
+PyYAML==3.11
+readline==6.2.4.1
+requests==2.24.0
+sendypy==4.0.9
+six==1.9.0
+twitter==1.14.3
+urllib3==1.25.10
+Werkzeug==0.9.4
diff --git a/weeklypedia/bake.py b/weeklypedia/bake.py
@@ -4,29 +4,30 @@
 import json
 from datetime import datetime, timedelta
 from os.path import dirname, join as pjoin
+from boltons.fileutils import mkdir_p
 
 from babel.dates import format_date
 from babel import UnknownLocaleError
 from dateutil.parser import parse as parse_date
 from ashes import TemplateNotFound
 
-from mail import Mailinglist, KEY
+from mail import sendy_send_campaign
 from fetch import get_latest_data_path
 
-from common import (DATA_BASE_PATH,
-                    DEFAULT_LANGUAGE,
+from common import (DEFAULT_LANGUAGE,
                     DEFAULT_INTRO,
                     DEBUG,
                     CUSTOM_INTRO_PATH,
                     LANG_MAP,
                     LOCAL_LANG_MAP,
                     SUBJECT_TMPL,
                     SUPPORTED_LANGS,
-                    SIGNUP_MAP,
-                    mkdir_p)
+                    SENDY_IDS)
 
 _CUR_PATH = dirname(os.path.abspath(__file__))
 
+ARCHIVE_URL = 'https://weekly.hatnote.com/archive/%s/index.html'
+
 INDEX_PATH = pjoin(dirname(_CUR_PATH), 'static', 'index.html')
 ARCHIVE_BASE_PATH = pjoin(dirname(_CUR_PATH), 'static', 'archive')
 ARCHIVE_PATH_TMPL = '{lang_shortcode}/{date_str}{dev_flag}/weeklypedia_{date_str}{dev_flag}{email_flag}.{fmt}'
@@ -71,14 +72,9 @@ def read_html(self):
     def read_text(self):
         return open(self.text_path).read()
 
-    def send(self, list_id, send_key):
-        mailinglist = Mailinglist(send_key + KEY)
-        mailinglist.new_campaign(self.subject,
-                                 self.read_html(),
-                                 self.read_text(),
-                                 list_id=list_id)
-        mailinglist.send_next_campaign()
-        return 'Success: sent issue %s' % self.lang
+    def send(self, list_id):
+        sendy_send_campaign(self.subject, self.read_text(), self.read_html(), list_id)
+        return 'Success: sent issue %s via sendy' % self.lang
 
 
 def get_past_issue_paths(lang, include_dev=False):
@@ -156,7 +152,6 @@ def bake_latest_issue(issue_ashes_env,
                       include_dev=DEBUG):
     ret = {'issues': []}
     issue_data = prep_latest_issue(lang, intro, include_dev)
-    issue_data['signup_url'] = SIGNUP_MAP[lang]
     # this fmt is used to generate the path, as well
     for fmt in ('html', 'json', 'txt', 'email'):
         rendered = render_issue(issue_data, issue_ashes_env, format=fmt)
@@ -181,6 +176,9 @@ def render_issue(render_ctx, issue_ashes_env,
     lang = render_ctx['short_lang_name']
     env = issue_ashes_env
     ctx = localize_data(render_ctx, lang)
+    ctx['list_id'] = SENDY_IDS[lang]
+    ctx['archive_link'] = ARCHIVE_URL % lang
+
     if format == 'html':
         ret = lang_fallback_render(env, lang, 'archive.html', ctx)
     elif format == 'email':
@@ -232,7 +230,7 @@ def render_archive(issue_ashes_env, lang):
         ret['issues'].insert(0, {'path': archive_path,
                                  'date': display_date})
     ret['lang'] = LANG_MAP[lang]
-    ret['signup_url'] = SIGNUP_MAP[lang]
+    ret['list_id'] = SENDY_IDS[lang]
     return issue_ashes_env.render('template_archive_index.html', ret)
 
 
@@ -253,6 +251,7 @@ def render_and_save_archives(issue_ashes_env):
     ret = []
     for lang in SUPPORTED_LANGS:
         out_path = ARCHIVE_INDEX_PATH_TMPL.format(lang_shortcode=lang)
+        mkdir_p(pjoin(ARCHIVE_BASE_PATH, lang))
         out_file = open(out_path, 'w')
         with out_file:
             rendered = render_archive(issue_ashes_env, lang)

diff --git a/weeklypedia/common.py b/weeklypedia/common.py
@@ -6,17 +6,18 @@
 
 DEBUG = False
 
+SENDY_URL = 'https://mailer.hatnote.com/s/'
+
 DEFAULT_LANGUAGE = 'en'
 DEFAULT_INTRO = 'Hello there! Welcome to our weekly digest of Wikipedia activity.'
 SUBJECT_TMPL = 'Weeklypedia {lang_name} #{issue_number}'
 
-DEBUG_LIST_ID = "a5ecbc7404"
-
 _CUR_PATH = dirname(os.path.abspath(__file__))
 
 LANG_MAP = json.load(open(pjoin(_CUR_PATH, 'language_codes.json')))
 LOCAL_LANG_MAP = {'en': u'English',
                   'de': u'Deutsch',
+                  'eo': u'Esperanto',
                   'fr': u'Français',
                   'ko': u'한국어',
                   'et': u'Eesti',
@@ -35,48 +36,40 @@
                   'oc': 'Occitan',
                   'ru': 'Russian',
                   'uk': 'Ukrainian'}
-SENDKEY = json.load(open(os.path.join(_CUR_PATH, 'secrets.json'))).get('key')
-SUPPORTED_LANGS = ['en', 'de', 'fr', 'ko', 'et', 'sv', 'da', 'it', 'ca', 'es',
-                   'fa', 'zh', 'ur', 'kn', 'lv', 'el', 'te', 'oc', 'ru', 'uk']
-API_BASE_URL = 'http://weeklypedia.toolforge.org/fetch/'
 
-ARCHIVE_BASE_PATH = pjoin(dirname(_CUR_PATH), 'static', 'archive')
+SENDY_KEY = json.load(open(os.path.join(_CUR_PATH, 'secrets.json'))).get('sendy_key')
+
+SENDY_IDS = {'ca': '9sW1OtFlCbJlYgxSXuahHQ',
+             'zh': 'aSQ6TT0VKpw0tmzBPaRZDg',
+             'da': '0cs1zVQp3892EjjL0763350TeQ',
+             #'en': 'ccIjgMNDQjgxlFR8MrQS3g', # load testing
+             #'en': 'VN7NFOFUPp5WjrbMSH7Puw', # debug
+             'en': 'rFf1E97OGw9qMfZh1F81KA',
+             'es': 'HTPao3LPmzm0UrCNxfBUgA',
+             'eo': 'H804892jOtJrNTukmVWOlrbA',
+             'et': 'db8mkJ2Tl6pnNUIIVfMFog',
+             'fr': 'ELz1OOSd3olC6LSCJmCqhw',
+             'de': 't0892Imxu8HTkzoPkrow11MQ',
+             'it': 'EkOruTQZ64fx7V5k9heZNw',
+             'kn': 'Dn9KffuyqLRKSY9XAwPCHQ',
+             'ko': '65Y8dYqreq2Frkav2WmJ9Q',
+             'lv': 'zgfaJH8Jskz7VxNai9zc763A',
+             'sv': 'bqpefw4ZBxMBHxrTz9dPKg',
+             'fa': 'mRGhgpBb4RnwDe25RtP8fA',
+             'el': 'Ts6mbUlmOCiD0mlWPL8T4A',
+             'oc': 'OrhmrHkNlTRR9KWNMAgDMQ',
+             'ru': 'IUDkAYoiJDQ7P3AQtLIAhQ',
+             'es': '5EzGTlwChgHME1TDa763nncA',
+             'te': 'rp4VkbQ1p2QXi560nIrF3w',
+             'ur': 'QiUlnjE3S9kPdvpzWQdK5Q',
+             'uk': 'gYfAtFPIVbJgVLkEfpO892Uw '}
+
+SUPPORTED_LANGS = SENDY_IDS.keys()
+
+API_BASE_URL = 'http://weeklypedia.toolforge.org/fetch/'
 
 DATA_BASE_PATH = pjoin(dirname(_CUR_PATH), 'static', 'data')
 DATA_PATH_TMPL = '{lang_shortcode}/{date_str}{dev_flag}/weeklypedia_{lang_shortcode}_{date_str}{dev_flag}.json'
 DATA_PATH_TMPL = pjoin(DATA_BASE_PATH, DATA_PATH_TMPL)
 
 CUSTOM_INTRO_PATH = pjoin(DATA_BASE_PATH, 'custom_intro.txt')
-
-SIGNUP_MAP = {'en': 'http://eepurl.com/MMlpX',
-              'de': 'http://eepurl.com/MMlG9',
-              'fr': 'http://eepurl.com/MMmVX',
-              'ko': 'http://eepurl.com/MMm8n',
-              'et': 'http://eepurl.com/MMnlf',
-              'sv': 'http://eepurl.com/MMTnP',
-              'da': 'http://eepurl.com/Sko4L',
-              'it': 'http://eepurl.com/MQTPb',
-              'ca': 'http://eepurl.com/M7HU9',
-              'eo': 'http://eepurl.com/_RE_X',
-              'es': 'http://eepurl.com/br8NNj',
-              'fa': 'http://eepurl.com/br8TlT',
-              'zh': 'http://eepurl.com/bsmGhT',
-              'ur': 'http://eepurl.com/bsmG1P',
-              'kn': 'http://eepurl.com/buxugf',
-              'lv': 'http://eepurl.com/b0V-yj',
-              'el': 'http://eepurl.com/b0WcUr',
-              'te': 'http://eepurl.com/b3EjMn',
-              'oc': 'http://eepurl.com/dxJIYf',
-              'ru': 'http://eepurl.com/dxJJGn',
-              'uk': 'http://eepurl.com/dxJKwf'}
-
-
-def mkdir_p(path):
-    # bolton
-    import errno
-    try:
-        os.makedirs(path)
-    except OSError as exc:
-        if exc.errno == errno.EEXIST and os.path.isdir(path):
-            return
-        raise
diff --git a/weeklypedia/fetch.py b/weeklypedia/fetch.py
@@ -4,10 +4,10 @@
 import json
 import urllib2
 from datetime import datetime
-
+from boltons.fileutils import mkdir_p
 
 from common import LANG_MAP, DEBUG, DEFAULT_LANGUAGE, API_BASE_URL
-from common import DATA_BASE_PATH, DATA_PATH_TMPL, mkdir_p
+from common import DATA_BASE_PATH, DATA_PATH_TMPL
 
 
 def fetch_rc(lang=DEFAULT_LANGUAGE):
@@ -75,7 +75,6 @@ def get_latest_data_path(lang, include_dev=DEBUG):
     return os.path.join(issue_path, latest_issue_fn)
 
 
-
 def get_argparser():
     from argparse import ArgumentParser
     desc = "fetch json data from labs"

diff --git a/weeklypedia/issue_templates/base/archive.html b/weeklypedia/issue_templates/base/archive.html
@@ -12,57 +12,73 @@
   <title>$HEAD_TITLE$</title>
 </head>
 <body>
-  <div class="header">
-    <h1><span>$BODY_TITLE$</span></h1>
-    <p>$DESCRIPTION$</p>
-    <p>$DELIVERY_DESCRIPTION$</p>
-  </div>
-  <div class="announce">
-    <ul class="breadcrumb">
-      <li><a href="../../../index.html">$HOME_LINK$</a></li>
-      <li><a href="../index.html">$ARCHIVE_LINK$</a></li>
-      <li>$ISSUE_LABEL$</li>
-    </ul>
-  </div>
-  <div class="main">
-    <div class="content">
-      <p id="intro">{intro|s}</p>
-      <h2 id="articles">$ARTICLES_HEADING$</h2>
-      <p>{#stats}$ARTICLES_SUMMARY${/stats} $ARTICLES_INTRO$:</p>
-      <ol>
-        {#mainspace}
-        <li><a href="https://{short_lang_name}.wikipedia.org/wiki/{title}" target="_blank">{title_s}</a> <span dir="ltr">(<a href="https://{short_lang_name}.wikipedia.org/w/index.php?title={title}&diff={rev_ids.new}&oldid={rev_ids.old}">$DIFF_COUNT$</a> $DIFF_AUTHOR_COUNT$)</span></li>
-        {/mainspace}
-      </ol>
+  <div class="container">
+    <div class="header">
+      <h1><span>$BODY_TITLE$</span></h1>
+      <p>$DESCRIPTION$</p>
+      <p>$DELIVERY_DESCRIPTION$</p>
+    </div>
+    <div class="announce">
+      <ul class="breadcrumb">
+        <li><a href="../../../index.html">$HOME_LINK$</a></li>
+        <li><a href="../index.html">$ARCHIVE_LINK$</a></li>
+        <li>$ISSUE_LABEL$</li>
+      </ul>
+    </div>
+    <div class="main">
+      <div class="content">
+        <p id="intro">{intro|s}</p>
+        <h2 id="articles">$ARTICLES_HEADING$</h2>
+        <p>{#stats}$ARTICLES_SUMMARY${/stats} $ARTICLES_INTRO$:</p>
+        <ol>
+          {#mainspace}
+          <li><a href="https://{short_lang_name}.wikipedia.org/wiki/{title}" target="_blank">{title_s}</a> <span dir="ltr">(<a href="https://{short_lang_name}.wikipedia.org/w/index.php?title={title}&diff={rev_ids.new}&oldid={rev_ids.old}">$DIFF_COUNT$</a> $DIFF_AUTHOR_COUNT$)</span></li>
+          {/mainspace}
+        </ol>
 
-      <h2 id="new-articles">$NEW_ARTICLES_HEADING$</h2>
-      <p>$NEW_ARTICLES_INTRO$</p>
-      <ol>
-        {#new_articles}
-        <li><a href="https://{short_lang_name}.wikipedia.org/wiki/{title}" target="_blank">{title_s}</a> <span dir="ltr">($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)</span></li>
-        {/new_articles}
-      </ol>
+        <h2 id="new-articles">$NEW_ARTICLES_HEADING$</h2>
+        <p>$NEW_ARTICLES_INTRO$</p>
+        <ol>
+          {#new_articles}
+          <li><a href="https://{short_lang_name}.wikipedia.org/wiki/{title}" target="_blank">{title_s}</a> <span dir="ltr">($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)</span></li>
+          {/new_articles}
+        </ol>
 
-      <h2 id="discussions">$DISCUSSIONS_HEADING$</h2>
-      <p>$DISCUSSIONS_INTRO$:</p>
-      <ol>
-        {#talkspace}
-        <li><a href="https://{short_lang_name}.wikipedia.org/wiki/Talk:{title}" target="_blank">{title_s}</a></li>
-        {/talkspace}
-      </ol>
+        <h2 id="discussions">$DISCUSSIONS_HEADING$</h2>
+        <p>$DISCUSSIONS_INTRO$:</p>
+        <ol>
+          {#talkspace}
+          <li><a href="https://{short_lang_name}.wikipedia.org/wiki/Talk:{title}" target="_blank">{title_s}</a></li>
+          {/talkspace}
+        </ol>
 
-      <h2 id="statistics">$STATS_HEADING$</h2>
-      <p>$STATS_INTRO$:</p>
-      <ul>
-        <li>$USER_EDIT_COUNTS$</li>
-        <li>$ANON_EDIT_COUNTS$</li>
-        <li>$BOT_EDIT_COUNTS$</li>
-      </ul>
-    <a href="{signup_url}" class="signup"><span>$SIGNUP_LINK$</span></a>
-    <div class="content">
-      <h2>$ABOUT_HEADING$</h2>
-      <p>$ABOUT_TEXT$</p>
-      <p><a href="http://blog.hatnote.com/">$BLOG_LINK$</a>.</p>
+        <h2 id="statistics">$STATS_HEADING$</h2>
+        <p>$STATS_INTRO$:</p>
+        <ul>
+          <li>$USER_EDIT_COUNTS$</li>
+          <li>$ANON_EDIT_COUNTS$</li>
+          <li>$BOT_EDIT_COUNTS$</li>
+        </ul>
+
+      <h2>$SIGNUP$</h2>
+      <p>$PITCH$</p>
+
+      <form action="https://mailer.hatnote.com/s/subscribe" method="POST" accept-charset="utf-8">
+        <label class="email-label" for="email">$EMAIL$</label>
+        <input class="email-input" type="email" name="email" id="email"/>
+        <div style="display:none;">
+          <label for="hp">HP</label><br/>
+          <input type="text" name="hp" id="hp"/></div>
+        <input type="hidden" name="list" value="{list_id}"/>
+        <input type="hidden" name="subform" value="yes"/>
+        <input class="sub-button" type="submit" name="submit" value="$SUBSCRIBE$" id="submit"/>
+      </form>
+
+      <div class="content">
+        <h2>$ABOUT_HEADING$</h2>
+        <p>$ABOUT_TEXT$</p>
+        <p><a href="http://blog.hatnote.com/">$BLOG_LINK$</a>.</p>
+      </div>
     </div>
   </div>
   <script>

diff --git a/weeklypedia/issue_templates/base/email.html b/weeklypedia/issue_templates/base/email.html
@@ -62,6 +62,29 @@ <h2 style="margin: 40px 0 10px;padding: 0;font: 800 Sans-Serif;font-size: 28px;l
                 <td style="margin: 0;padding: 0;font-family: Sans-Serif;font-size: 100%;line-height: 1.6;"></td>
         </tr>
 </table>
+
+<table border="0" cellpadding="0" cellspacing="0" width="100%" style="background-color:#ffffff;border-top:1px solid #e5e5e5">
+            <tbody><tr>
+                <td align="center" valign="top" style="padding-top:20px;padding-bottom:20px">
+                    <table border="0" cellpadding="0" cellspacing="0">
+                        <tbody><tr>
+                            <td align="center" valign="top" style="color:#606060;font-family:Helvetica,Arial,sans-serif;font-size:11px;line-height:150%;padding-right:20px;padding-bottom:5px;padding-left:20px;text-align:center">
+                                $SENT_TO$ [Email]
+                                <br>
+                                <a href="{archive_link}" style="color:#404040!important" target="_blank">$SIGNUP$</a>  · <a href="[unsubscribe]" style="color:#404040!important" target="_blank">$UNSUBSCRIBE$</a>
+                                <br>
+                                Hatnote · 558 N 3rd St. · San Jose, CA 95112 · USA
+                                <br>
+                                <br>
+
+                            </td>
+                        </tr>
+                    </tbody></table>
+                </td>
+            </tr>
+        </tbody>
+</table>
+
 <!-- /body -->
 
 </body>