Skip to content

Commit 496c948

Browse files
AAndyProgramAAndyProgram
committed
2023.11.15.0
ADD FACEBOOK SiteSettingsBase: update 'CLONE_PROPERTIES' function (exclude 'DoNotUse' attribute) API.Instagram: handle 401 error API.ThreadsNet.SiteSettings: make the class compatible for Facebook xHanster, XVideos, PornHub, ThisVid: update download function for search queries Hosts.PropertyValueHost: set the 'Exists' value based on the 'DoNotUse' attribute Hosts.SettingsHost: use 'GetObjectMembers' instead of 'GetTypeInfo.DeclaredMembers' to get class members
1 parent 96705f1 commit 496c948

22 files changed

Lines changed: 1124 additions & 59 deletions

File tree

‎SCrawler.PluginProvider/Attributes/Attributes.vb‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ Namespace Plugin.Attributes
6565
End Class
6666
''' <summary>Attribute to disable some properties for host use</summary>
6767
<AttributeUsage(AttributeTargets.Property, AllowMultiple:=False, Inherited:=False)> Public NotInheritable Class DoNotUse : Inherits Attribute
68+
Public ReadOnly Value As Boolean = True
69+
Public Sub New()
70+
End Sub
71+
Public Sub New(ByVal Value As Boolean)
72+
Me.Value = Value
73+
End Sub
6874
End Class
6975
''' <summary>Special property updater</summary>
7076
<AttributeUsage(AttributeTargets.Method, AllowMultiple:=True, Inherited:=False)> Public NotInheritable Class PropertyUpdater : Inherits Attribute

‎SCrawler/API/Base/DeclaredNames.vb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ Namespace API.Base
1111
Friend Const Header_Authorization As String = "authorization"
1212
Friend Const Header_CSRFToken As String = "x-csrf-token"
1313

14+
Friend Const Header_FB_FRIENDLY_NAME As String = "x-fb-friendly-name"
15+
1416
Friend Const ConcurrentDownloadsCaption As String = "Concurrent downloads"
1517
Friend Const ConcurrentDownloadsToolTip As String = "The number of concurrent downloads."
1618
Friend Const SavedPostsUserNameCaption As String = "Saved posts user"

‎SCrawler/API/Base/SiteSettingsBase.vb‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,9 +278,13 @@ Namespace API.Base
278278
'1 = clone
279279
'2 = any
280280
Dim filterUC As Func(Of MemberInfo, Byte, Boolean) = Function(ByVal m As MemberInfo, ByVal __mode As Byte) As Boolean
281-
With m.GetCustomAttribute(Of PClonableAttribute)
282-
Return Not .Self Is Nothing AndAlso (__mode = 2 OrElse If(__mode = 0, .Update, .Clone))
283-
End With
281+
If m.GetCustomAttribute(Of DoNotUse) Is Nothing Then
282+
Return False
283+
Else
284+
With m.GetCustomAttribute(Of PClonableAttribute)
285+
Return Not .Self Is Nothing AndAlso (__mode = 2 OrElse If(__mode = 0, .Update, .Clone))
286+
End With
287+
End If
284288
End Function
285289
Dim filterAll As Func(Of MemberInfo, Boolean) = Function(m) filterUC.Invoke(m, 2)
286290
Dim filterC As Func(Of MemberInfo, Boolean) = Function(m) If(Full, filterAll.Invoke(m), filterUC.Invoke(m, 1))

‎SCrawler/API/Facebook/Declarations.vb‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
2+
' This program is free software: you can redistribute it and/or modify
3+
' it under the terms of the GNU General Public License as published by
4+
' the Free Software Foundation, either version 3 of the License, or
5+
' (at your option) any later version.
6+
'
7+
' This program is distributed in the hope that it will be useful,
8+
' but WITHOUT ANY WARRANTY
9+
Imports System.Text.RegularExpressions
10+
Imports PersonalUtilities.Functions.XML.Base
11+
Imports PersonalUtilities.Functions.RegularExpressions
12+
Namespace API.Facebook
13+
Friend Module Declarations
14+
Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
15+
Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
16+
Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
17+
18+
Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue)
19+
Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue)
20+
Friend ReadOnly Regex_ProfileUrlID As RParams = RParams.DMS("profile.php\?id=(\d+)", 1, EDP.ReturnValue)
21+
Friend ReadOnly Regex_VideoPageID As RParams = RParams.DMS("pageid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
22+
Friend ReadOnly Regex_StoryBucket As RParams = RParams.DMS("story_bucket[^\>]*?(\d+)", 1, EDP.ReturnValue)
23+
24+
Friend ReadOnly Regex_VideoIDFromURL As RParams = RParams.DMS("facebook.com/([^/]+/videos/|watch/\D*[\?&]{1}v=)(\d+)", 2, EDP.ReturnValue)
25+
Friend ReadOnly Regex_PostHtmlFullPicture As RParams = RParams.DM("^((?!_[ps]{1}\d+x\d+).)*$", 0, EDP.ReturnValue)
26+
27+
Friend ReadOnly SpecialNode() As NodeParams = {New NodeParams("attachment", True, True, True, True, 30),
28+
New NodeParams("media", True, True, True, True, 0),
29+
New NodeParams("photo_image", True, True, True, True, 0),
30+
New NodeParams("uri", True, True, True, True, 0)}
31+
Friend ReadOnly SpecialNode2() As NodeParams = {New NodeParams("result", True, True, True, True, 30),
32+
New NodeParams("data", True, True, True, True, 0),
33+
New NodeParams("currmedia", True, True, True, True, 0),
34+
New NodeParams("image", True, True, True, True, 0),
35+
New NodeParams("uri", True, True, True, True, 0)}
36+
End Module
37+
End Namespace

‎SCrawler/API/Facebook/SiteSettings.vb‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
2+
' This program is free software: you can redistribute it and/or modify
3+
' it under the terms of the GNU General Public License as published by
4+
' the Free Software Foundation, either version 3 of the License, or
5+
' (at your option) any later version.
6+
'
7+
' This program is distributed in the hope that it will be useful,
8+
' but WITHOUT ANY WARRANTY
9+
Imports SCrawler.API.Base
10+
Imports SCrawler.Plugin
11+
Imports SCrawler.Plugin.Attributes
12+
Imports PersonalUtilities.Tools.Web.Clients
13+
Imports PersonalUtilities.Functions.RegularExpressions
14+
Namespace API.Facebook
15+
<Manifest("AndyProgram_Facebook"), SavedPosts, SeparatedTasks(1), SpecialForm(False)>
16+
Friend Class SiteSettings : Inherits ThreadsNet.SiteSettings
17+
#Region "Declarations"
18+
#Region "Auth"
19+
<PropertyOption(AllowNull:=False, ControlText:="Accept", ControlToolTip:="Header 'Accept'", IsAuth:=True), ControlNumber(21), PXML, PClonable>
20+
Friend ReadOnly Property Header_Accept As PropertyValue
21+
<PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True)>
22+
Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue
23+
Get
24+
Return __HH_IG_APP_ID
25+
End Get
26+
End Property
27+
<DoNotUse> Friend Overrides ReadOnly Property HH_CSRF_TOKEN As PropertyValue
28+
Get
29+
Return __HH_CSRF_TOKEN
30+
End Get
31+
End Property
32+
<PropertyOption(ControlText:="sec-ch-ua-platform-ver", ControlToolTip:="sec-ch-ua-platform-version", IsAuth:=True, LeftOffset:=120), ControlNumber(51), PXML, PClonable>
33+
Friend ReadOnly Property HH_PLATFORM_VER As PropertyValue
34+
#End Region
35+
#Region "Defaults"
36+
<PropertyOption(ControlText:="Download photos", IsAuth:=False), PXML, PClonable>
37+
Friend ReadOnly Property ParsePhotoBlock As PropertyValue
38+
<PropertyOption(ControlText:="Download videos", IsAuth:=False), PXML, PClonable>
39+
Friend ReadOnly Property ParseVideoBlock As PropertyValue
40+
<PropertyOption(ControlText:="Download stories", IsAuth:=False), PXML, PClonable>
41+
Friend ReadOnly Property ParseStoriesBlock As PropertyValue
42+
#End Region
43+
#End Region
44+
#Region "Initializer"
45+
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
46+
MyBase.New("Facebook", "facebook.com", AccName, Temp, My.Resources.SiteResources.FacebookIcon_32, My.Resources.SiteResources.FacebookPic_37)
47+
48+
With Responser.Headers
49+
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.facebook.com"))
50+
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.facebook.com"))
51+
.Remove(DeclaredNames.Header_FB_FRIENDLY_NAME)
52+
End With
53+
Header_Accept = New PropertyValue(String.Empty, GetType(String))
54+
HH_PLATFORM_VER = New PropertyValue(String.Empty, GetType(String))
55+
ParsePhotoBlock = New PropertyValue(True)
56+
ParseVideoBlock = New PropertyValue(True)
57+
ParseStoriesBlock = New PropertyValue(True)
58+
59+
UrlPatternUser = "https://www.facebook.com/{0}"
60+
UserRegex = RParams.DMS("facebook.com/(profile.php\?id=\d+|[^\?&/]+)", 1)
61+
ImageVideoContains = "facebook.com"
62+
UserOptionsType = GetType(UserExchangeOptions)
63+
End Sub
64+
#End Region
65+
#Region "GetInstance"
66+
Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider
67+
Return New UserData
68+
End Function
69+
#End Region
70+
#Region "UpdateResponserData"
71+
Friend Overrides Sub UpdateResponserData(ByVal Resp As Responser)
72+
With Responser.Cookies
73+
.Update(Resp.Cookies)
74+
If .Changed Then Responser.SaveCookies() : .Changed = False
75+
End With
76+
End Sub
77+
#End Region
78+
#Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
79+
Friend Overrides Function BaseAuthExists() As Boolean
80+
Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value)
81+
End Function
82+
Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
83+
Return DirectCast(User, UserData).GetProfileUrl
84+
End Function
85+
Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
86+
Return Media.URL_BASE
87+
End Function
88+
Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions
89+
Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL)
90+
If e.Exists Then
91+
e.Options = e.UserName
92+
Dim v$ = RegexReplace(e.UserName, Regex_ProfileUrlID)
93+
If Not v.IsEmptyString Then
94+
e.UserName = v
95+
Else
96+
e.UserName = e.UserName.StringRemoveWinForbiddenSymbols
97+
End If
98+
End If
99+
Return e
100+
End Function
101+
Friend Overrides Function IsMyImageVideo(ByVal URL As String) As ExchangeOptions
102+
If Not URL.IsEmptyString AndAlso Not CStr(AConvert(Of String)(URL, Regex_VideoIDFromURL, String.Empty)).IsEmptyString Then
103+
Return New ExchangeOptions(Site, String.Empty) With {.Exists = True}
104+
Else
105+
Return Nothing
106+
End If
107+
End Function
108+
#End Region
109+
End Class
110+
End Namespace

0 commit comments

Comments
 (0)