slothbot/index_docs.py at main · FeatureBaseDB/slothbot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import sys
import time
import requests

from bs4 import BeautifulSoup
import nltk

from lib.database import weaviate_schema, weaviate_update
from lib.ai import ai

nltk.download('punkt')
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

# weavaite schema
schema = weaviate_schema("docs")

# get the sitemap and load it
sitemap = "https://docs.featurebase.com/sitemap.xml"
result = requests.get(sitemap).text

# parse with BS
soup = BeautifulSoup(result, features="xml")
urls = soup.find_all("loc")

# build list of URLs
_urls = []
for url in urls:
	_urls.append(url.text)

# overwrite the list
urls = _urls

for url in urls:
	if "404" in url:
		continue

	res = requests.get(url)
	html_page = res.content
	soup = BeautifulSoup(html_page, 'html.parser')

	title = soup.find('title').text

	print("Indexing: %s" % title)

	text = soup.find("div", {"id": "main-content"})

	words = ""
	previous_words = ""
	for i, entry in enumerate(tokenizer.tokenize(text.text)):
		print(i)
		words = words + " " + entry.replace("\n", " ")

		if (i % 3) == 0:
			# ai_doc = ai("mirror", {"words": words})

			document = {
				"sentence": words,
				"url": url,
				"title": title
			}

			print("==================")
			print(document)

			for x in range(10):
				try:
					print(weaviate_update(document, "Docs"))
					break
				except Exception as ex:
					print("%s - sleeping 10 seconds for retry" % ex)
					time.sleep(10)
			print("==================")
			words = ""

	if words != "":
		# ai_doc = ai("mirror", {"words": words})

		document = {
			"sentence": words,
			"url": url,
			"title": title
		}

		print("==================")
		print(document)

		for x in range(10):
			try:
				print(weaviate_update(document, "Docs"))
				break
			except Exception as ex:
				print("%s - sleeping 10 seconds for retry" % ex)
				time.sleep(10)
		print("==================")

sys.exit()

"""

	splits = text.text.split(" ") # split on spaces

	fulltexts = []
	_fulltext = ""
	for split in splits:
		_fulltext = _fulltext + split + " "

		# if we have a long string and a stop character
		if len(_fulltext) > 208 and (split.endswith(".") or split.endswith('."') or split.endswith("?") or split.endswith("!")):
			fulltexts.append(_fulltext)
			_fulltext = ""

	fulltexts.append(_fulltext)

	for fulltext in fulltexts:
		document = {
			"url": url,
			"title": title,
			"sentence": fulltext.strip(" ")
		}
		weaviate_update(document, "Docs")
"""