更新數據插入

Author: ColdSalad

AuthorizeManage.dll

@@ -394,7 +394,7 @@ async def on_search(self):
             'search_content': search_content,
             'search_count': search_count,
             'crawl_count': crawl_count,
-            'device_id': device_id,
+            'device': device_id,
             'combo_value': combo_value,
             'action': 'search',
             'types': 'facebook'

facebook/AuthorizeManage.dll

@@ -10,15 +10,14 @@
 import shutil  # 适用于Linux和macOS
 import sys
 import re
-import aiohttp
-import requests
+import datetime
 from urllib.parse import urlparse, parse_qs
 from PyQt5.QtWidgets import QApplication
 from FB_loginwin import win_main
 from playwright.async_api import async_playwright
 from FB_status import StatusWindow
 from database_manager import db_manager
-
+from concurrent.futures import ThreadPoolExecutor
 
 class Crawler:
     def __init__(self, cookies, params):
@@ -40,6 +39,8 @@ def __init__(self, cookies, params):
         self.supportId = ""
         self.post_user_cunt = 0
         self.post_name = ""
+        self.eq_type = None
+        self.device = params.get('device')
 
     async def safe_update_status(self, text):
         """安全的异步状态更新"""
@@ -527,7 +528,7 @@ async def getusers(self):
         if not addresses:
             print("没有提供地址列表")
             return
-
+        self.eq_type = 1
         for i in range(len(addresses)):
             url = addresses[i].strip()
             # 创建一个新的CSV文件名
@@ -545,6 +546,8 @@ async def getusers(self):
                 groups_num_selector = '//span[@dir="auto"]/div//a[@role="link" and contains(text(), "成員")]'
                 groups_num_function = await self.page.wait_for_selector(groups_num_selector, timeout=10000)
                 self.groups_num = await groups_num_function.inner_text()
+                day_match = re.search(r'\d+', self.groups_num)
+                self.groups_num = day_match.group()
                 print(self.groups_num)
                 await self.robust_update_status(f"社團名:{self.groups_name} 社團人數：{self.groups_num}")
             except Exception as e:
@@ -590,7 +593,7 @@ async def getusers(self):
                                 'name': text.strip(),
                                 'user_id': user_id
                             })
-                            print(f"{user_counter}：{user_id} {text.strip()}")
+                            # print(f"{user_counter}：{user_id} {text.strip()}")
                             await self.robust_update_status(f"{user_counter}：{user_id} {text.strip()}")
                             in_csv_data.append([user_id, text.strip(), self.extract_group_id(url)])
 
@@ -616,20 +619,42 @@ async def getusers(self):
                 if scroll_attempts >= 3:  # 连续3次没有新用户就停止
                     print("已加载所有用户")
                     break
-            # 将数据写入CSV文件
-            with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
-                csv_writer = csv.writer(csvfile)
-                csv_writer.writerow(['userid', 'username', 'societiesid'])  # 写入表头
-                csv_writer.writerows(in_csv_data)  # 写入数据
-            print(f"爬取完成，共获取 {user_counter} 个用户信息")
+            if len(users) > 0:
+                # 将数据写入CSV文件
+                with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
+                    csv_writer = csv.writer(csvfile)
+                    csv_writer.writerow(['userid', 'username', 'societiesid'])  # 写入表头
+                    csv_writer.writerows(in_csv_data)  # 写入数据
+                print(f"爬取完成，共获取 {user_counter} 个用户信息")
+                # 使用线程池执行数据库提交（避免阻塞主线程）
+                loop = asyncio.get_event_loop()
+                with ThreadPoolExecutor() as executor:
+                    await loop.run_in_executor(
+                        executor,
+                        submit_data_to_database,
+                        csv_filename,
+                        i,
+                        self.eq_type,
+                        self.extract_group_id(url),  # 社团ID
+                        self.groups_name,  # 社团名称
+                        self.groups_num,  # 总成员数
+                        user_counter  # 实际获取数量
+                    )
+
+                await self.robust_update_status(f"{csv_filename}数据提交完成")
+                upend_time = datetime.datetime.now()
+                db_manager.update_updata_table(self.device, len(users), upend_time)
+            else:
+                print("無用戶")
+
             # return users
 
     async def getusers_fans(self):
         addresses = self.params.get('addresses', [])
         if not addresses:
             print("没有提供地址列表")
             return
-
+        self.eq_type = 2
         for i in range(len(addresses)):
             url = addresses[i].strip()
             # 创建一个新的CSV文件名
@@ -703,7 +728,7 @@ async def getusers_fans(self):
                                 'name': text.strip(),
                                 'user_id': user_id
                             })
-                            print(f"{user_counter}：{user_id} {text.strip()}")
+                            # print(f"{user_counter}：{user_id} {text.strip()}")
                             await self.robust_update_status(f"{user_counter}：{user_id} {text.strip()}")
                             in_csv_data.append([user_id, text.strip(), await self.extract_facebook_identifier(url)])
 
@@ -729,12 +754,32 @@ async def getusers_fans(self):
                 if scroll_attempts >= 3:  # 连续3次没有新用户就停止
                     print("已加载所有用户")
                     break
-            # 将数据写入CSV文件
-            with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
-                csv_writer = csv.writer(csvfile)
-                csv_writer.writerow(['userid', 'username', 'societiesid'])  # 写入表头
-                csv_writer.writerows(in_csv_data)  # 写入数据
-            print(f"爬取完成，共获取 {user_counter} 个用户信息")
+            if len(users) > 0:
+                # 将数据写入CSV文件
+                with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
+                    csv_writer = csv.writer(csvfile)
+                    csv_writer.writerow(['userid', 'username', 'societiesid'])  # 写入表头
+                    csv_writer.writerows(in_csv_data)  # 写入数据
+                print(f"爬取完成，共获取 {user_counter} 个用户信息")
+                # 使用线程池执行数据库提交（避免阻塞主线程）
+                loop = asyncio.get_event_loop()
+                with ThreadPoolExecutor() as executor:
+                    await loop.run_in_executor(
+                        executor,
+                        submit_data_to_database,
+                        csv_filename,
+                        i,
+                        self.eq_type,
+                        self.extract_fans_id(url),  # 粉丝专页ID
+                        self.groups_name,  # 粉丝专页名称
+                        self.groups_num,  # 总粉丝数
+                        user_counter  # 实际获取数量
+                    )
+                await self.robust_update_status(f"{csv_filename}数据提交完成")
+                upend_time = datetime.datetime.now()
+                db_manager.update_updata_table(self.device, len(users), upend_time)
+            else:
+                print("無用戶")
 
     async def getusers_like(self):
         await self.page.goto(url="https://www.facebook.com/", wait_until='load', timeout=50000)
@@ -987,6 +1032,8 @@ async def get_sponsor_user(self):
             }
             print('提交数据', post_info)
             db_manager.insert_post_info(post_info)
+            upend_time = datetime.datetime.now()
+            db_manager.update_updata_table(self.device, len(users), upend_time)
         else:
             print('数据为空不提交')
             await self.robust_update_status(f"沒有成員跳過~")
@@ -1206,6 +1253,56 @@ async def force_minimize_browser(self):
         self.minimize_browser_window()
 
 
+def submit_data_to_database(csv_filename, batch_number, eq_type, societies_url_id, societies_name, total_number,
+                            getnum):
+    """提交数据到数据库"""
+    try:
+        with open(csv_filename, 'r', newline='', encoding='utf-8') as csvfile:
+            csv_reader = csv.reader(csvfile)
+            next(csv_reader)  # 跳过表头
+            data = list(csv_reader)
+            print(f"批次 {batch_number}: 开始提交数据...")
+
+            batch_size = 80  # 每批80条
+            for i in range(0, len(data), batch_size):
+                batch_data = data[i:i + batch_size]
+
+                if eq_type == 1:
+                    # 插入社团用户表
+                    success = db_manager.insert_societies_user_batch(batch_data)
+                else:
+                    # 插入粉丝用户表
+                    success = db_manager.insert_fans_user_batch(batch_data)
+
+                if not success:
+                    print(f"批次 {batch_number} 第 {i // batch_size + 1} 批数据插入失败")
+                    continue
+
+                print(f"批次 {batch_number} 第 {i // batch_size + 1} 批数据插入成功")
+
+                # 批次间等待
+                sleep_time = random.uniform(8, 14)
+                time.sleep(sleep_time)
+
+            print(f"批次 {batch_number}: 提交完成，共 {len(data)} 条数据")
+
+            # 所有批次数据提交完成后，插入汇总信息
+            if eq_type == 1:
+                # 插入社团汇总信息
+                db_manager.insert_societies_inf(societies_url_id, societies_name, total_number, getnum)
+            else:
+                # 插入粉丝专页汇总信息
+                db_manager.insert_fans_inf(societies_url_id, societies_name, total_number, getnum)
+
+        # 提交完成后删除CSV文件
+        os.remove(csv_filename)
+        print(f"完成: {csv_filename}")
+        return True
+
+    except Exception as e:
+        print(f"提交数据到数据库时出错: {str(e)}")
+        return False
+
 def parse_bool(type_data):
     type_data = str(type_data).lower().strip()
     return type_data in ('true', '1', 'yes', 'yes')

facebook/FB_win.py

@@ -151,7 +151,7 @@ def check_authorization():
         connection_string = (
             r"Driver={SQL Server};"
             r"Server=dbs.kydb.vip;"
-            r"Database=LINEGroupSend;"
+            r"Database=DeviceAuthData;"
             r"UID=sa;"
             r"PWD=Yunsin@#861123823_shp4;"
             r"timeout=35;"
@@ -163,7 +163,7 @@ def check_authorization():
 
         query = """
                 SELECT PCCoded, installDate, ExpiryDate
-                FROM FBUserData
+                FROM UserData
                 WHERE PCCoded = ? \
                 """
         cursor.execute(query, (machine_code,))

facebook/FBmain.py

@@ -230,5 +230,134 @@ def insert_post_info(self, post_info: any) -> bool:
                     pass
             return False
 
+    def insert_societies_user_batch(self, data):
+        """批量插入社团用户数据"""
+        try:
+            conn = self.get_connection()
+            cursor = conn.cursor()
+
+            # 使用executemany批量插入数据
+            cursor.executemany(
+                "INSERT INTO societiesUser (userid, username, societiesid) VALUES (?, ?, ?)",
+                data
+            )
+            conn.commit()
+            print(f"成功插入 {len(data)} 条社团用户数据")
+            return True
+        except Exception as e:
+            print(f"插入社团用户数据时出错: {e}")
+            if conn:
+                conn.rollback()
+            return False
+        finally:
+            if cursor:
+                cursor.close()
+
+    def insert_fans_user_batch(self, data):
+        """批量插入粉丝用户数据"""
+        try:
+            conn = self.get_connection()
+            cursor = conn.cursor()
+
+            # 使用executemany批量插入数据
+            cursor.executemany(
+                "INSERT INTO FansUser (userid, username, societiesid) VALUES (?, ?, ?)",
+                data
+            )
+            conn.commit()
+            print(f"成功插入 {len(data)} 条粉丝用户数据")
+            return True
+        except Exception as e:
+            print(f"插入粉丝用户数据时出错: {e}")
+            if conn:
+                conn.rollback()
+            return False
+        finally:
+            if cursor:
+                cursor.close()
+
+
+    def insert_societies_inf(self, societiesid, societiesname, number, getnum):
+        """插入社团信息到 societiesInf 表"""
+        try:
+            conn = self.get_connection()
+            cursor = conn.cursor()
+
+            cursor.execute(
+                "INSERT INTO societiesInf (societiesid, societiesname, number, getnum) VALUES (?, ?, ?, ?)",
+                societiesid, societiesname, number, getnum
+            )
+            conn.commit()
+            print(f"成功插入社团信息: {societiesid} - {societiesname}")
+            return True
+        except Exception as e:
+            print(f"插入社团信息失败: {e}")
+            if conn:
+                conn.rollback()
+            return False
+        finally:
+            if cursor:
+                cursor.close()
+
+
+    def insert_fans_inf(self, userid, fansname, number, getnum):
+        """插入粉丝专页信息到 FansInf 表"""
+        try:
+            conn = self.get_connection()
+            cursor = conn.cursor()
+
+            cursor.execute(
+                "INSERT INTO FansInf (userid, fansname, number, getnum) VALUES (?, ?, ?, ?)",
+                userid, fansname, number, getnum
+            )
+            conn.commit()
+            print(f"成功插入粉丝专页信息: {userid} - {fansname}")
+            return True
+        except Exception as e:
+            print(f"插入粉丝专页信息失败: {e}")
+            if conn:
+                conn.rollback()
+            return False
+        finally:
+            if cursor:
+                cursor.close()
+
+    def update_updata_table(self, device_name, updatanumber, uptime):
+        """
+        更新 upData 表
+        使用 MERGE 语句，如果同一天有同一个设备，则更新数量（累加），否则插入新记录
+
+        参数:
+            device_name: 设备名称
+            updatanumber: 更新数据数量
+            uptime: 更新时间
+        """
+        try:
+            conn = self.get_connection()
+            cursor = conn.cursor()
+
+            cursor.execute("""
+                MERGE INTO upData AS target
+                USING (VALUES (?, ?, ?)) AS source(deviceName, updatanumber, uptime)
+                ON target.deviceName = source.deviceName AND CAST(target.uptime AS DATE) = CAST(source.uptime AS DATE)
+                WHEN MATCHED THEN
+                    UPDATE SET target.updatanumber = target.updatanumber + source.updatanumber,
+                               target.uptime = source.uptime
+                WHEN NOT MATCHED THEN
+                    INSERT (deviceName, updatanumber, uptime)
+                    VALUES (source.deviceName, source.updatanumber, source.uptime);
+            """, device_name, updatanumber, uptime)
+
+            conn.commit()
+            print(f"成功更新 upData 表: {device_name} - {updatanumber} - {uptime}")
+            return True
+        except Exception as e:
+            print(f"更新 upData 表失败: {e}")
+            if conn:
+                conn.rollback()
+            return False
+        finally:
+            if cursor:
+                cursor.close()
 # 创建全局实例
 db_manager = DatabaseManager()

facebook/authorization.py

@@ -87,7 +87,7 @@ def _init_ui(self):
         version_label.setAlignment(Qt.AlignHCenter)
         nav_layout.addWidget(version_label)
 
-        day_label = QLabel(f"剩余天數: {self.day}")
+        day_label = QLabel(f"剩余: {self.day} 天")
         day_label.setFont(QFont("微軟雅黑", 8))
         day_label.setStyleSheet("color: #999;")
         day_label.setAlignment(Qt.AlignHCenter)