Merge pull request #182 from BCSDLab/develop

Master <- Develop

Author: Choon0414

crawling/koreatech_article/article.py

@@ -8,9 +8,10 @@
 from bs4 import BeautifulSoup, Comment
 import urllib3
 import pymysql
+import uuid
 
 from delete_article import delete_article
-from table import replace_table
+from table import replace_table, upload_txt
 from login_v2 import login
 from login import get_jwt_token
 from slack_notice import filter_nas, notice_to_slack
@@ -285,6 +286,11 @@ def crawling_article(board: Board, host: str, url: str) -> Article:
     # 표 처리
     # content = replace_table(content, board, num)
 
+    # content s3 업로드 및 url 삽입
+    random_uuid = str(uuid.uuid4().hex)
+    file_name = f'articles/content/board_{board.id}/{random_uuid}.txt'
+    content = upload_txt(file_name=file_name, text_content=content)
+
     # ===== 첨부 파일 =====
     attachment = list(map(
         lambda tag: Attachment(

crawling/koreatech_article/convert_content.py

@@ -0,0 +1,78 @@
+import pymysql
+import urllib3
+import uuid
+
+from config import MYSQL_CONFIG
+from table import upload_txt
+
+
+def connect_db():
+    urllib3.disable_warnings()
+    conn = pymysql.connect(host=MYSQL_CONFIG['host'],
+                           port=MYSQL_CONFIG['port'],
+                           user=MYSQL_CONFIG['user'],
+                           password=MYSQL_CONFIG['password'],
+                           db=MYSQL_CONFIG['db'],
+                           charset='utf8')
+    return conn
+
+
+def convert_content_to_url(connection):
+    cur = connection.cursor()
+    batch_size = 500
+    last_id = 0
+    total_articles = 0
+
+    try:
+        while True:
+            cur.execute("""
+                SELECT `id`, `board_id`, `content`
+                FROM `new_articles`
+                WHERE `content` IS NOT NULL
+                    AND `board_id` != 14
+                    AND `id` > %s
+                ORDER BY `id` ASC
+                LIMIT %s
+            """, (last_id, batch_size))
+
+            articles = cur.fetchall()
+            if not articles:
+                break
+
+            for article in articles:
+                article_id, board_id, content = article
+                random_uuid = str(uuid.uuid4().hex)
+                file_name = f'articles/content/board_{board_id}/{random_uuid}.txt'
+                content_url = upload_txt(file_name=file_name, text_content=content)
+
+                update_cur = connection.cursor()
+                update_cur.execute("""
+                    UPDATE `new_articles`
+                    SET `content` = %s
+                    WHERE `id` = %s
+                """, (content_url, article_id))
+                print(f"article {article_id} url: {content_url}")
+                update_cur.close()
+
+                total_articles += 1
+
+            last_id = articles[-1][0]
+            connection.commit()
+
+    except Exception as error:
+        connection.rollback()
+        print(error)
+    finally:
+        cur.close()
+        print(f"total articles: {total_articles}")
+
+
+if __name__ == "__main__":
+    connection = None
+    try:
+        connection = connect_db()
+        convert_content_to_url(connection)
+    except Exception as error:
+        print(error)
+    finally:
+        connection.close()

crawling/koreatech_article/table.py

@@ -105,3 +105,27 @@ def upload_image(s3, file_name: str, image: bytes) -> str:
         print(e)
 
     return f'{S3_CONFIG["upload_domain"]}/{file_name}'
+
+
+def upload_txt(file_name: str, text_content: str) -> str:
+    s3 = boto3.client(
+        service_name='s3',
+        aws_access_key_id=S3_CONFIG['aws_access_key_id'],
+        aws_secret_access_key=S3_CONFIG['aws_secret_access_key'],
+    )
+    encoded_text = text_content.encode('utf-8')
+
+    try:
+        s3.upload_fileobj(
+            Fileobj=BytesIO(encoded_text),
+            Bucket=S3_CONFIG['bucket'],
+            Key=file_name,
+            ExtraArgs={
+                'ContentType': 'text/plain; charset=utf-8',
+                'ACL': 'public-read'
+            }
+        )
+    except Exception as e:
+        print(e)
+
+    return f'{S3_CONFIG["upload_domain"]}/{file_name}'