From 849e57f45067b5cc9290bd933dbbed79e85c75c5 Mon Sep 17 00:00:00 2001 From: gauthiier Date: Sat, 24 Aug 2019 13:06:59 +0200 Subject: [PATCH] new lists --- .gitignore | 2 +- archive/archive.py | 82 ++++++++++++++++++++++++++++++++-------------- archive/sql.py | 4 ++- archive/util.py | 2 +- www/routes.py | 2 ++ 5 files changed, 65 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 49fd8f2..969a0f7 100755 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ archives/ config/ config.py -test.py +test*.py #macos .DS_Store diff --git a/archive/archive.py b/archive/archive.py index 8965e08..41e4841 100644 --- a/archive/archive.py +++ b/archive/archive.py @@ -53,37 +53,58 @@ def connect_db(database, host, user, password): except mariadb.Error as error: print("Error: {}".format(error)) if error.errno == 1049: - if util.y_n_question("Table " + archive_name + " does not exist. Create it?"): - print("creating") - else: - print("not creating") + print("Database " + database + " does not exist.") return None finally: return con +def list_tables_db(database, host, user, password): + + con = connect_db(database, host, user, password) + if con is not None: + + try: + cursor = con.cursor() + cursor.execute(archive.sql.SHOW_TABLE) + + results = [] + for t in cursor: + results.append(t[0]) + return results + + except mariadb.Error as error: + print("Error: {}".format(error)) + finally: + cursor.close() + con.close() + + + + + class Archive: data = None # "raw" json data db_con = None - def __init__(self, archive_name, archive_dir): + def __init__(self, archive_name, archive_dir_or_db_config): - if isinstance(archive_name, str): + if isinstance(archive_dir_or_db_config, str): # need a filename or a dir name.... print("reading archive " + archive_name, end='') + archive_dir = archive_dir_or_db_config (self.data, self.archive_name) = load_from_file(archive_name, archive_name, archive_dir) print(" - done.") + elif isinstance(archive_dir_or_db_config, dict): + self.archive_name = archive_name + self.db_con = connect_db(config['database'], config['host'], config['user'], config['password']) - def __init__(self, archive_name, database, host, user, password): + # def __init__(self, archive_name, database, host, user, password): - self.archive_name = archive_name - self.db_con = connect_db(database, host, user, password) + # self.archive_name = archive_name + # self.db_con = connect_db(database, host, user, password) - def __init__(self, archive_name, config): - - self.archive_name = archive_name - self.db_con = connect_db(config['database'], config['host'], config['user'], config['password']) def __enter__(self): return self @@ -92,12 +113,15 @@ class Archive: if self.db_con is not None: self.db_con.close() - - def create_db(self, host, database, user, password): + def create_db(self, config=None): print("creating table: " + self.archive_name, end='') - self.db_con = connect_db(database, host, user, password) if self.db_con is None: + if config is not None: + self.db_con = connect_db(config['database'], config['host'], config['user'], config['password']) + + if self.db_con is None: + print(" - no connection... Aborting.") return try: @@ -110,12 +134,15 @@ class Archive: print(" - done.") - def insert_db(self, host, database, user, password): - self.db_con = connect_db(database, host, user, password) + def insert_db(self, config=None): if self.db_con is None: - return + if config is not None: + self.db_con = connect_db(config['database'], config['host'], config['user'], config['password']) + + if self.db_con is None: + return try: cursor = self.db_con.cursor() @@ -136,8 +163,8 @@ class Archive: self.db_con.commit() except mariadb.Error as error: - pass - # print("Error: {}".format(error)) + print("Error: {}".format(error)) + pass finally: cursor.close() @@ -153,11 +180,14 @@ class Archive: date_ = archive.util.format_date(m, self.archive_name) if date_ is None or from_ is None: - # print("\nerrorororororo") - # print(m['from'] + " -- " + m['date']) + print("\nerrorororororo") + print(m['from'] + " -- " + m['date']) continue - cursor.execute(archive.sql.INSERT, (from_,author_name_,to_,m["subject"],date_,m["content-type"],m["content"],m["url"])) + + str_insert = archive.sql.INSERT.format(self.archive_name) + cursor.execute(str_insert, (from_, author_name_,to_, m["subject"], date_, m["content-type"], m["content"], m["url"])) + # cursor.execute(archive.sql.INSERT.format(self.archive_name, from_, author_name_,to_, m["subject"], date_, m["content-type"], m["content"], m["url"])) n_inserted += 1 if "follow-up" in m: @@ -168,6 +198,10 @@ class Archive: #duplication continue <------------------------- look this up... # print("\nError: {}".format(error)) continue + else: + print("\nError: {}".format(error)) + print(str_insert) + continue return n_inserted diff --git a/archive/sql.py b/archive/sql.py index eee0474..46a1a25 100644 --- a/archive/sql.py +++ b/archive/sql.py @@ -12,7 +12,7 @@ CREATE = "CREATE TABLE `{}` (" \ "FULLTEXT (`from_`, `author_name_`)" \ ") ENGINE = InnoDB;" -INSERT = ("INSERT INTO nettime_l" +INSERT = ("INSERT INTO {}" "(from_, author_name_, to_, subject_, date_, content_type_, content_, url_) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)") @@ -28,4 +28,6 @@ FROM_QUERY_BOOLEAN = ("SELECT from_, author_name_, subject_, date_, url_ FROM {} FROM_QUERY_NL = ("SELECT from_, author_name_, subject_, date_, url_ FROM {} " "WHERE MATCH(from_, author_name_) AGAINST('{}') ORDER BY date_") +SHOW_TABLE = "show tables" + # SELECT from_, author_name_, subject_, date_, url_ FROM nettime_l WHERE MATCH(content_) AGAINST('%s' IN BOOLEAN MODE) \ No newline at end of file diff --git a/archive/util.py b/archive/util.py index 70a9d67..ca4ede1 100755 --- a/archive/util.py +++ b/archive/util.py @@ -79,7 +79,7 @@ def format_from(msg): def format_to(msg): if "to" not in msg or msg["to"] is None: - return None + return "n/a" to_str = msg["to"] toks = email.utils.parseaddr(to_str) diff --git a/www/routes.py b/www/routes.py index 952e80f..b4df1bf 100644 --- a/www/routes.py +++ b/www/routes.py @@ -20,6 +20,8 @@ def favicon(): def searh(): if len(request.args) < 1: + # q: list all table or keep predefined wconfig.lists_to_serve? + wconfig.lists_to_serve = archive.show_tables_db(config.db['database'], config.db['host'], config.db['user'], config.db['password']) return render_template("search.html", archives=wconfig.lists_to_serve, fields=['content', 'from']) k_arg = request.args.get('keyword')