evandro/pagodera
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add TibiaWiki scraper project and update .gitignore

Browse Source 
Integrates a Scrapy-based TibiaWiki scraper (spider, pipelines, SQLite/Images settings, README). Updates .gitignore to exclude generated data directories (/iconScript/data, /data/images) to keep the repository clean.
This commit is contained in:
Evandro Fonseca Junior committed 2025-10-30 00:55:25 -03:00
1 parent bf12f2c6f1
commit 29ba5aece0
27 files changed
+1702 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

iconScript/explore_structure.py
+8 -4
View File
@@ -14,14 +14,18 @@ ITEMS_URL = f"{BASE_URL}/wiki/Itens"
def fetch_page(url):
"""Busca o conteúdo HTML de uma página"""
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1'
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Cache-Control': 'max-age=0'
}
response = requests.get(url, headers=headers, timeout=30)
response = requests.get(url, headers=headers, timeout=30, allow_redirects=True)
response.raise_for_status()
return response.text