Merge pull request #176 from BCSDLab/feature/convert_article_content_to_s3_url

feat: article content s3 업로드 및 url로 변경

Author: Choon0414

crawling/koreatech_article/convert_content.py

@@ -0,0 +1,79 @@
+import pymysql
+import urllib3
+import uuid
+
+from crawling.config import MYSQL_CONFIG
+from table import upload_txt
+
+
+def connect_db():
+    urllib3.disable_warnings()
+    conn = pymysql.connect(host=MYSQL_CONFIG['host'],
+                           port=MYSQL_CONFIG['port'],
+                           user=MYSQL_CONFIG['user'],
+                           password=MYSQL_CONFIG['password'],
+                           db=MYSQL_CONFIG['db'],
+                           charset='utf8')
+    return conn
+
+
+def convert_content_to_url(connection):
+    cur = connection.cursor()
+    batch_size = 500
+    last_id = 0
+    total_articles = 0
+
+    while True:
+        try:
+            cur.execute("""
+                SELECT `id`, `board_id`, `content`
+                FROM `new_articles`
+                WHERE `content` IS NOT NULL
+                    AND `board_id` != 14
+                    AND `id` > %s
+                ORDER BY `id` ASC
+                LIMIT %s
+            """, (last_id, batch_size))
+
+            articles = cur.fetchall()
+            if not articles:
+                break
+
+            for article in articles:
+                article_id, board_id, content = article
+                random_uuid = str(uuid.uuid4().hex)
+                file_name = f'articles/content/board_{board_id}/{random_uuid}.txt'
+                content_url = upload_txt(file_name=file_name, text_content=content)
+
+                update_cur = connection.cursor()
+                update_cur.execute("""
+                    UPDATE `new_articles`
+                    SET `content` = %s
+                    WHERE `id` = %s
+                """, (content_url, article_id))
+                print(f"article {article_id} url: {content_url}")
+                update_cur.close()
+
+                total_articles += 1
+
+            last_id = articles[-1]['id']
+
+            connection.commit()
+
+        except Exception as error:
+            connection.rollback()
+            print(error)
+        finally:
+            cur.close()
+            print(f"total articles: {total_articles}")
+
+
+if __name__ == "__main__":
+    connection = None
+    try:
+        connection = connect_db()
+        convert_content_to_url(connection)
+    except Exception as error:
+        print(error)
+    finally:
+        connection.close()

crawling/koreatech_article/table.py

@@ -105,3 +105,26 @@ def upload_image(s3, file_name: str, image: bytes) -> str:
         print(e)
 
     return f'{S3_CONFIG["upload_domain"]}/{file_name}'
+
+def upload_txt(file_name: str, text_content: str) -> str:
+    s3 = boto3.client(
+        service_name='s3',
+        aws_access_key_id=S3_CONFIG['aws_access_key_id'],
+        aws_secret_access_key=S3_CONFIG['aws_secret_access_key'],
+    )
+    encoded_text = text_content.encode('utf-8')
+
+    try:
+        s3.upload_fileobj(
+            Fileobj=BytesIO(encoded_text),
+            Bucket=S3_CONFIG['bucket'],
+            Key=file_name,
+            ExtraArgs={
+                'ContentType': 'text/plain; charset=utf-8',
+                'ACL': 'public-read'
+            }
+        )
+    except Exception as e:
+        print(e)
+
+    return f'{S3_CONFIG["upload_domain"]}/{file_name}'