Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Carrying over some updates from my test tool setup #6

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Carrying over some updates from my test tool setup #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
830ef1f
Updated version
Samwalton9 Jan 19, 2018
9827a51
Update to latest lithoxyl function names
Samwalton9 Jan 19, 2018
f76d97b
Check if logs folder exists, make it if not.
Samwalton9 Jan 19, 2018
5af77d9
Fix a couple of bugs (they already appear to be fixed on production)
Samwalton9 Jan 19, 2018
a51ccdc
Revert to live db number
Samwalton9 Jan 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions dal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ def db_connect(db, host, read_default_file=DB_CONFIG_PATH):
return connection

def wiki_db_connect(lang):
wiki_db_name = self.lang + 'wiki_p'
wiki_db_host = self.lang + WIKI_DB_DOMAIN
wiki_db_name = lang + 'wiki_p'
wiki_db_host = lang + 'wiki.' + WIKI_DB_DOMAIN
return db_connect(wiki_db_name, wiki_db_host)


Expand Down
11 changes: 7 additions & 4 deletions log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import os

from lithoxyl import Logger, SensibleSink, Formatter, StreamEmitter
from lithoxyl import Logger, SensibleSink, SensibleFormatter, StreamEmitter, SensibleFilter
from lithoxyl.emitters import FileEmitter
from lithoxyl.filters import ThresholdFilter

class FixedFileEmitter(FileEmitter):
def __init__(self, filepath, encoding=None, **kwargs):
Expand All @@ -11,12 +10,16 @@ def __init__(self, filepath, encoding=None, **kwargs):

CUR_PATH = os.path.dirname(os.path.abspath(__file__))
LOG_FILE_PATH = os.path.join(CUR_PATH, 'logs', 'update_log.txt')
LOG_FOLDER_PATH = os.path.join(CUR_PATH, 'logs')

if not os.path.isdir(LOG_FOLDER_PATH):
os.mkdir(LOG_FOLDER_PATH)

tlog = Logger('toplog')

file_fmt = Formatter('{status_char}{end_local_iso8601_noms_notz} - {duration_secs}s - {record_name} - {message}')
file_fmt = SensibleFormatter('{status_char}{end_local_iso8601_noms_notz} - {duration_secs}s - {record_name} - {message}')
file_emt = FixedFileEmitter(LOG_FILE_PATH)
file_filter = ThresholdFilter(success='critical',
file_filter = SensibleFilter(success='critical',
failure='info',
exception='debug')
file_sink = SensibleSink(formatter=file_fmt,
Expand Down
239 changes: 107 additions & 132 deletions schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,132 +1,107 @@

-- begin run logs

CREATE TABLE start_log (start_log_id integer not null auto_increment,
start_timestamp TIMESTAMP,
lang varchar(8),
command varchar(1024),
run_uuid varchar(36),
PRIMARY KEY (start_log_id),
KEY (start_timestamp),
KEY (lang));

CREATE TABLE complete_log (complete_log_id integer not null auto_increment,
complete_timestamp TIMESTAMP,
lang varchar(8),
output varchar(4096),
run_uuid varchar(36),
PRIMARY KEY (complete_log_id),
KEY (complete_timestamp),
KEY (lang));

-- end run logs

CREATE TABLE /*_*/hashtags (
ht_id int NOT NULL PRIMARY KEY AUTO_INCREMENT,
ht_text varbinary(767) NOT NULL default '',
ht_create_timestamp varbinary(14) NOT NULL default '',
ht_update_timestamp varbinary(14) NOT NULL default '');

CREATE INDEX /*i*/ht_text ON hashtags (ht_text);
CREATE INDEX /*i*/ht_create_timestamp ON hashtags (ht_create_timestamp);

CREATE TABLE /*_*/mentions (
mn_id int NOT NULL PRIMARY KEY AUTO_INCREMENT,
mn_text varbinary(767) NOT NULL default '',
mn_create_timestamp varbinary(14) NOT NULL default '',
mn_update_timestamp varbinary(14) NOT NULL default '');

CREATE INDEX /*i*/mn_text ON mentions (mn_text);
CREATE INDEX /*i*/mn_create_timestamp ON mentions (mn_create_timestamp);

CREATE TABLE /*_*/mention_recentchanges (
mn_id int NOT NULL,
mnrc_id int NOT NULL);

CREATE TABLE /*_*/hashtag_recentchanges (
ht_id int NOT NULL,
htrc_id int NOT NULL);

CREATE UNIQUE INDEX /*i*/htrc_hashtag_recentchanges ON /*_*/hashtag_recentchanges (ht_id, htrc_id);
CREATE INDEX /*i*/htrc_hashtag ON /*_*/hashtag_recentchanges (ht_id);

CREATE TABLE /*_*/recentchanges (
htrc_id int NOT NULL PRIMARY KEY AUTO_INCREMENT,
htrc_lang varchar(32) NOT NULL default '',
rc_id int NOT NULL,
rc_timestamp varbinary(14) NOT NULL default '',
rc_cur_time varbinary(14) NOT NULL default '',

-- As in revision
rc_user int unsigned NOT NULL default 0,
rc_user_text varchar(255) binary NOT NULL,

-- When pages are renamed, their RC entries do _not_ change.
rc_namespace int NOT NULL default 0,
rc_title varchar(255) binary NOT NULL default '',

-- as in revision...
rc_comment varbinary(767) NOT NULL default '',
rc_minor tinyint unsigned NOT NULL default 0,

-- Edits by user accounts with the 'bot' rights key are
-- marked with a 1 here, and will be hidden from the
-- default view.
rc_bot tinyint unsigned NOT NULL default 0,

-- Set if this change corresponds to a page creation
rc_new tinyint unsigned NOT NULL default 0,

-- Key to page_id (was cur_id prior to 1.5).
-- This will keep links working after moves while
-- retaining the at-the-time name in the changes list.
rc_cur_id int unsigned NOT NULL default 0,

-- rev_id of the given revision
rc_this_oldid int unsigned NOT NULL default 0,

-- rev_id of the prior revision, for generating diff links.
rc_last_oldid int unsigned NOT NULL default 0,

-- The type of change entry (RC_EDIT,RC_NEW,RC_LOG,RC_EXTERNAL)
rc_type tinyint unsigned NOT NULL default 0,

-- The source of the change entry (replaces rc_type)
-- default of '' is temporary, needed for initial migration
rc_source varchar(16) binary not null default '',

-- If the Recent Changes Patrol option is enabled,
-- users may mark edits as having been reviewed to
-- remove a warning flag on the RC list.
-- A value of 1 indicates the page has been reviewed.
rc_patrolled tinyint unsigned NOT NULL default 0,

-- Recorded IP address the edit was made from, if the
-- $wgPutIPinRC option is enabled.
rc_ip varbinary(40),

-- Text length in characters before
-- and after the edit
rc_old_len int,
rc_new_len int,

-- Visibility of recent changes items, bitfield
rc_deleted tinyint unsigned NOT NULL default 0,

-- Value corresponding to log_id, specific log entries
rc_logid int unsigned NOT NULL default 0,
-- Store log type info here, or null
rc_log_type varbinary(255) NULL default NULL,
-- Store log action or null
rc_log_action varbinary(255) NULL default NULL,
-- Log params
rc_params blob NULL
);

CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp);
CREATE INDEX /*i*/rc_lang_namespace_title ON /*_*/recentchanges (htrc_lang, rc_namespace, rc_title);
CREATE INDEX /*i*/rc_lang_cur_id ON /*_*/recentchanges (htrc_lang, rc_cur_id);
CREATE INDEX /*i*/lang_new_name_timestamp ON /*_*/recentchanges (htrc_lang, rc_new,rc_namespace,rc_timestamp);
CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip);
CREATE INDEX /*i*/rc_lang_ns_usertext ON /*_*/recentchanges (htrc_lang, rc_namespace, rc_user_text);
CREATE INDEX /*i*/rc_lang_user_text ON /*_*/recentchanges (htrc_lang, rc_user_text, rc_timestamp);
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `complete_log` (
`complete_log_id` int(11) NOT NULL AUTO_INCREMENT,
`complete_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`lang` varbinary(8) DEFAULT NULL,
`output` varbinary(4096) DEFAULT NULL,
`run_uuid` varbinary(36) DEFAULT NULL,
PRIMARY KEY (`complete_log_id`),
KEY `complete_timestamp` (`complete_timestamp`),
KEY `lang` (`lang`)
) ENGINE=Aria AUTO_INCREMENT=1514514 DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `hashtag_recentchanges` (
`ht_id` int(11) NOT NULL,
`htrc_id` int(11) NOT NULL,
UNIQUE KEY `htrc_hashtag_recentchanges` (`ht_id`,`htrc_id`),
KEY `htrc_hashtag` (`ht_id`)
) ENGINE=Aria DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `hashtags` (
`ht_id` int(11) NOT NULL AUTO_INCREMENT,
`ht_text` varbinary(767) NOT NULL DEFAULT '',
`ht_create_timestamp` varbinary(14) NOT NULL DEFAULT '',
`ht_update_timestamp` varbinary(14) NOT NULL DEFAULT '',
PRIMARY KEY (`ht_id`),
KEY `ht_text` (`ht_text`),
KEY `ht_create_timestamp` (`ht_create_timestamp`)
) ENGINE=Aria AUTO_INCREMENT=36146 DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `mention_recentchanges` (
`mn_id` int(11) NOT NULL,
`mnrc_id` int(11) NOT NULL
) ENGINE=Aria DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `mentions` (
`mn_id` int(11) NOT NULL AUTO_INCREMENT,
`mn_text` varbinary(767) NOT NULL DEFAULT '',
`mn_create_timestamp` varbinary(14) NOT NULL DEFAULT '',
`mn_update_timestamp` varbinary(14) NOT NULL DEFAULT '',
PRIMARY KEY (`mn_id`),
KEY `mn_text` (`mn_text`),
KEY `mn_create_timestamp` (`mn_create_timestamp`)
) ENGINE=Aria AUTO_INCREMENT=129 DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `recentchanges` (
`htrc_id` int(11) NOT NULL AUTO_INCREMENT,
`htrc_lang` varbinary(32) NOT NULL DEFAULT '',
`rc_id` int(11) NOT NULL,
`rc_timestamp` varbinary(14) NOT NULL DEFAULT '',
`rc_cur_time` varbinary(14) NOT NULL DEFAULT '',
`rc_user` int(10) unsigned NOT NULL DEFAULT '0',
`rc_user_text` varbinary(255) NOT NULL,
`rc_namespace` int(11) NOT NULL DEFAULT '0',
`rc_title` varbinary(255) NOT NULL DEFAULT '',
`rc_comment` varbinary(767) NOT NULL DEFAULT '',
`rc_minor` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_bot` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_new` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_cur_id` int(10) unsigned NOT NULL DEFAULT '0',
`rc_this_oldid` int(10) unsigned NOT NULL DEFAULT '0',
`rc_last_oldid` int(10) unsigned NOT NULL DEFAULT '0',
`rc_type` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_source` varbinary(16) NOT NULL DEFAULT '',
`rc_patrolled` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_ip` varbinary(40) DEFAULT NULL,
`rc_old_len` int(11) DEFAULT NULL,
`rc_new_len` int(11) DEFAULT NULL,
`rc_deleted` tinyint(3) unsigned NOT NULL DEFAULT '0',
`rc_logid` int(10) unsigned NOT NULL DEFAULT '0',
`rc_log_type` varbinary(255) DEFAULT NULL,
`rc_log_action` varbinary(255) DEFAULT NULL,
`rc_params` blob,
PRIMARY KEY (`htrc_id`),
KEY `rc_timestamp` (`rc_timestamp`),
KEY `rc_lang_namespace_title` (`htrc_lang`,`rc_namespace`,`rc_title`),
KEY `rc_lang_cur_id` (`htrc_lang`,`rc_cur_id`),
KEY `lang_new_name_timestamp` (`htrc_lang`,`rc_new`,`rc_namespace`,`rc_timestamp`),
KEY `rc_ip` (`rc_ip`),
KEY `rc_lang_ns_usertext` (`htrc_lang`,`rc_namespace`,`rc_user_text`),
KEY `rc_lang_user_text` (`htrc_lang`,`rc_user_text`,`rc_timestamp`),
KEY `rc_id_htrc_lang` (`rc_id`,`htrc_lang`) COMMENT 'speed up add_recentchange 20160305'
) ENGINE=Aria AUTO_INCREMENT=5566734 DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `start_log` (
`start_log_id` int(11) NOT NULL AUTO_INCREMENT,
`start_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`lang` varbinary(8) DEFAULT NULL,
`command` varbinary(1024) DEFAULT NULL,
`run_uuid` varbinary(36) DEFAULT NULL,
PRIMARY KEY (`start_log_id`),
KEY `start_timestamp` (`start_timestamp`),
KEY `lang` (`lang`)
) ENGINE=Aria AUTO_INCREMENT=1529298 DEFAULT CHARSET=binary PAGE_CHECKSUM=1;
/*!40101 SET character_set_client = @saved_cs_client */;