From c5c653d97d9d3f08088c423a39311e000f755cd9 Mon Sep 17 00:00:00 2001 From: momo Date: Tue, 10 Aug 2021 14:38:40 +0300 Subject: [PATCH 1/5] Add start feature --- tinylinks/management/commands/import_yourls_db.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tinylinks/management/commands/import_yourls_db.py b/tinylinks/management/commands/import_yourls_db.py index ec3a71f..c261586 100644 --- a/tinylinks/management/commands/import_yourls_db.py +++ b/tinylinks/management/commands/import_yourls_db.py @@ -4,7 +4,7 @@ import mysql.connector from django.core.management.base import BaseCommand -from tinylinks.management.commands import _config, _queries +from tinylinks.management.commands import _config from tinylinks.models import Tinylink, TinylinkLog @@ -23,7 +23,7 @@ def get_tinylinks_query_data(self, start) -> List[tuple]: return data def insert_tinylinks(self): - start = 0 + start = self.start data = self.get_tinylinks_query_data(start) while data: print("Processing rows from {} to {}".format(start, start + self.chunk_length)) @@ -48,7 +48,7 @@ def get_tinylinks_logs_query_data(self, start) -> List[tuple]: return data def insert_tinylinks_logs(self): - start = 0 + start = self.start data = self.get_tinylinks_logs_query_data(start) while data: print("Processing rows from {} to {}".format(start, @@ -71,6 +71,7 @@ def add_arguments(self, parser): parser.add_argument("password", nargs=1, type=str) parser.add_argument("dbname", nargs=1, type=str) parser.add_argument("chunk-length", nargs="?", type=int, default=100) + parser.add_argument("start", nargs="?", type=int, default=0) def handle(self, *args, **options): _config.set_configs( @@ -79,5 +80,6 @@ def handle(self, *args, **options): database=options["dbname"][0], ) self.chunk_length = options.get("chunk-length") + self.start = options.get("start") self.insert_tinylinks() self.insert_tinylinks_logs() From 7fd08ed950a2539f1d38dc6cdbeb493b98c6c19d Mon Sep 17 00:00:00 2001 From: momo Date: Tue, 10 Aug 2021 15:54:51 +0300 Subject: [PATCH 2/5] Add begin feature --- .../management/commands/import_yourls_db.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tinylinks/management/commands/import_yourls_db.py b/tinylinks/management/commands/import_yourls_db.py index c261586..d276eec 100644 --- a/tinylinks/management/commands/import_yourls_db.py +++ b/tinylinks/management/commands/import_yourls_db.py @@ -8,15 +8,15 @@ from tinylinks.models import Tinylink, TinylinkLog -TINYLINK_QUERY = "SELECT url, keyword FROM yourls_url LIMIT %s, %s;" -TINYLINKLOG_QUERY = "SELECT referrer, user_agent, ip_address, click_time FROM yourls_log LIMIT %s, %s;" +TINYLINK_QUERY = "SELECT url, keyword FROM yourls_url WHERE id > %s LIMIT %s, %s;" +TINYLINKLOG_QUERY = "SELECT referrer, user_agent, ip_address, click_time FROM yourls_log WHERE id > %s LIMIT %s, %s;" class Command(BaseCommand): - def get_tinylinks_query_data(self, start) -> List[tuple]: + def get_tinylinks_query_data(self, begin, start) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINK_QUERY, (start, self.chunk_length)) + cursor.execute(TINYLINK_QUERY, (begin, start, self.chunk_length)) data = [(long_url.decode('utf-8'), short_url) for (long_url, short_url) in cursor] cnx.close() cursor.close() @@ -24,7 +24,7 @@ def get_tinylinks_query_data(self, start) -> List[tuple]: def insert_tinylinks(self): start = self.start - data = self.get_tinylinks_query_data(start) + data = self.get_tinylinks_query_data(self.begin, start) while data: print("Processing rows from {} to {}".format(start, start + self.chunk_length)) tinylinks_to_add = [ @@ -33,12 +33,12 @@ def insert_tinylinks(self): ] Tinylink.objects.bulk_create(tinylinks_to_add) start += self.chunk_length - data = self.get_tinylinks_query_data(start) + data = self.get_tinylinks_query_data(self.begin, start) - def get_tinylinks_logs_query_data(self, start) -> List[tuple]: + def get_tinylinks_logs_query_data(self, begin, start) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINKLOG_QUERY, (start, self.chunk_length)) + cursor.execute(TINYLINKLOG_QUERY, (begin, start, self.chunk_length)) data = [ (referrer, user_agent, ip_address, click_time) for (referrer, user_agent, ip_address, click_time) in cursor @@ -49,7 +49,7 @@ def get_tinylinks_logs_query_data(self, start) -> List[tuple]: def insert_tinylinks_logs(self): start = self.start - data = self.get_tinylinks_logs_query_data(start) + data = self.get_tinylinks_logs_query_data(self.begin, start) while data: print("Processing rows from {} to {}".format(start, start + self.chunk_length)) @@ -64,14 +64,15 @@ def insert_tinylinks_logs(self): ] TinylinkLog.objects.bulk_create(tinylinks_logs_to_add) start += self.chunk_length - data = self.get_tinylinks_logs_query_data(start) + data = self.get_tinylinks_logs_query_data(self.begin, start) def add_arguments(self, parser): parser.add_argument("username", nargs=1, type=str) parser.add_argument("password", nargs=1, type=str) parser.add_argument("dbname", nargs=1, type=str) - parser.add_argument("chunk-length", nargs="?", type=int, default=100) parser.add_argument("start", nargs="?", type=int, default=0) + parser.add_argument("begin", nargs="?", type=int, default=0) + parser.add_argument("chunk-length", nargs="?", type=int, default=100) def handle(self, *args, **options): _config.set_configs( @@ -79,7 +80,8 @@ def handle(self, *args, **options): password=options["password"][0], database=options["dbname"][0], ) - self.chunk_length = options.get("chunk-length") self.start = options.get("start") + self.begin = options.get("begin") + self.chunk_length = options.get("chunk-length") self.insert_tinylinks() self.insert_tinylinks_logs() From 3481ab6eb0fdd38c58c70e2dff56bc0156aad41e Mon Sep 17 00:00:00 2001 From: momo Date: Tue, 10 Aug 2021 16:27:50 +0300 Subject: [PATCH 3/5] Change the order of chunk-length and start --- tinylinks/management/commands/import_yourls_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinylinks/management/commands/import_yourls_db.py b/tinylinks/management/commands/import_yourls_db.py index d276eec..dffa27d 100644 --- a/tinylinks/management/commands/import_yourls_db.py +++ b/tinylinks/management/commands/import_yourls_db.py @@ -16,7 +16,7 @@ class Command(BaseCommand): def get_tinylinks_query_data(self, begin, start) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINK_QUERY, (begin, start, self.chunk_length)) + cursor.execute(TINYLINK_QUERY, (begin, self.chunk_length, start)) data = [(long_url.decode('utf-8'), short_url) for (long_url, short_url) in cursor] cnx.close() cursor.close() @@ -38,7 +38,7 @@ def insert_tinylinks(self): def get_tinylinks_logs_query_data(self, begin, start) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINKLOG_QUERY, (begin, start, self.chunk_length)) + cursor.execute(TINYLINKLOG_QUERY, (begin, self.chunk_length, start)) data = [ (referrer, user_agent, ip_address, click_time) for (referrer, user_agent, ip_address, click_time) in cursor From f0c764feb55c01f50d9d7b16d546d4c0aed0ee55 Mon Sep 17 00:00:00 2001 From: momo Date: Wed, 11 Aug 2021 08:57:29 +0300 Subject: [PATCH 4/5] Refactor :begin to start_id , start as offset. Add a helping text for the command. --- .../management/commands/import_yourls_db.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tinylinks/management/commands/import_yourls_db.py b/tinylinks/management/commands/import_yourls_db.py index dffa27d..3481129 100644 --- a/tinylinks/management/commands/import_yourls_db.py +++ b/tinylinks/management/commands/import_yourls_db.py @@ -13,32 +13,32 @@ class Command(BaseCommand): - def get_tinylinks_query_data(self, begin, start) -> List[tuple]: + def get_tinylinks_query_data(self, start_id, offset) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINK_QUERY, (begin, self.chunk_length, start)) + cursor.execute(TINYLINK_QUERY, (start_id, self.chunk_length, offset)) data = [(long_url.decode('utf-8'), short_url) for (long_url, short_url) in cursor] cnx.close() cursor.close() return data def insert_tinylinks(self): - start = self.start - data = self.get_tinylinks_query_data(self.begin, start) + offset = self.offset + data = self.get_tinylinks_query_data(self.start_id, offset) while data: - print("Processing rows from {} to {}".format(start, start + self.chunk_length)) + print("Processing rows from {} to {}".format(offset, offset + self.chunk_length)) tinylinks_to_add = [ Tinylink(long_url=long_url, short_url=shorturl) for long_url, shorturl in data ] Tinylink.objects.bulk_create(tinylinks_to_add) - start += self.chunk_length - data = self.get_tinylinks_query_data(self.begin, start) + offset += self.chunk_length + data = self.get_tinylinks_query_data(self.start_id, offset) - def get_tinylinks_logs_query_data(self, begin, start) -> List[tuple]: + def get_tinylinks_logs_query_data(self, start_id, offset) -> List[tuple]: cnx = mysql.connector.connect(**_config.config) cursor = cnx.cursor() - cursor.execute(TINYLINKLOG_QUERY, (begin, self.chunk_length, start)) + cursor.execute(TINYLINKLOG_QUERY, (start_id, self.chunk_length, offset)) data = [ (referrer, user_agent, ip_address, click_time) for (referrer, user_agent, ip_address, click_time) in cursor @@ -48,11 +48,11 @@ def get_tinylinks_logs_query_data(self, begin, start) -> List[tuple]: return data def insert_tinylinks_logs(self): - start = self.start - data = self.get_tinylinks_logs_query_data(self.begin, start) + offset = self.offset + data = self.get_tinylinks_logs_query_data(self.start_id, offset) while data: - print("Processing rows from {} to {}".format(start, - start + self.chunk_length)) + print("Processing rows from {} to {}".format(offset, + offset + self.chunk_length)) tinylinks_logs_to_add = [ TinylinkLog( referrer=referrer, @@ -63,16 +63,16 @@ def insert_tinylinks_logs(self): for referrer, user_agent, remote_ip, datetime in data ] TinylinkLog.objects.bulk_create(tinylinks_logs_to_add) - start += self.chunk_length - data = self.get_tinylinks_logs_query_data(self.begin, start) + offset += self.chunk_length + data = self.get_tinylinks_logs_query_data(self.start_id, offset) def add_arguments(self, parser): - parser.add_argument("username", nargs=1, type=str) - parser.add_argument("password", nargs=1, type=str) - parser.add_argument("dbname", nargs=1, type=str) - parser.add_argument("start", nargs="?", type=int, default=0) - parser.add_argument("begin", nargs="?", type=int, default=0) - parser.add_argument("chunk-length", nargs="?", type=int, default=100) + parser.add_argument("username", nargs=1, type=str, help="Database username") + parser.add_argument("password", nargs=1, type=str, help="Database user password") + parser.add_argument("dbname", nargs=1, type=str, help="Database name") + parser.add_argument("offset", nargs="?", type=int, default=0, help="The offset of processing rows") + parser.add_argument("start_id", nargs="?", type=int, default=0, help="The starting id of imported rows") + parser.add_argument("chunk-length", nargs="?", type=int, default=100, help="The chunk length") def handle(self, *args, **options): _config.set_configs( @@ -80,8 +80,8 @@ def handle(self, *args, **options): password=options["password"][0], database=options["dbname"][0], ) - self.start = options.get("start") - self.begin = options.get("begin") + self.offset = options.get("offset") + self.start_id = options.get("start_id") self.chunk_length = options.get("chunk-length") self.insert_tinylinks() self.insert_tinylinks_logs() From 0a869d793c7645aea9a69c40199e738ac13b3cf1 Mon Sep 17 00:00:00 2001 From: momo Date: Wed, 11 Aug 2021 11:39:09 +0300 Subject: [PATCH 5/5] fix --- tinylinks/management/commands/import_yourls_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinylinks/management/commands/import_yourls_db.py b/tinylinks/management/commands/import_yourls_db.py index 3481129..fbae231 100644 --- a/tinylinks/management/commands/import_yourls_db.py +++ b/tinylinks/management/commands/import_yourls_db.py @@ -71,7 +71,7 @@ def add_arguments(self, parser): parser.add_argument("password", nargs=1, type=str, help="Database user password") parser.add_argument("dbname", nargs=1, type=str, help="Database name") parser.add_argument("offset", nargs="?", type=int, default=0, help="The offset of processing rows") - parser.add_argument("start_id", nargs="?", type=int, default=0, help="The starting id of imported rows") + parser.add_argument("start-id", nargs="?", type=int, default=0, help="The starting id of imported rows") parser.add_argument("chunk-length", nargs="?", type=int, default=100, help="The chunk length") def handle(self, *args, **options): @@ -81,7 +81,7 @@ def handle(self, *args, **options): database=options["dbname"][0], ) self.offset = options.get("offset") - self.start_id = options.get("start_id") + self.start_id = options.get("start-id") self.chunk_length = options.get("chunk-length") self.insert_tinylinks() self.insert_tinylinks_logs()