feat: add TibiaWiki scraper project and update .gitignore
Integrates a Scrapy-based TibiaWiki scraper (spider, pipelines, SQLite/Images settings, README). Updates .gitignore to exclude generated data directories (/iconScript/data, /data/images) to keep the repository clean.
This commit is contained in:
1 parent
bf12f2c6f1
commit
29ba5aece0
27 files changed
+1702
-4
No files matched your search
@@ -0,0 +1,74 @@
|
||||
# Scrapy settings for tibiawiki project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://docs.scrapy.org/en/latest/topics/settings.html
|
||||
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
BOT_NAME = 'tibiawiki'
|
||||
|
||||
SPIDER_MODULES = ['tibiawiki.spiders']
|
||||
NEWSPIDER_MODULE = 'tibiawiki.spiders'
|
||||
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = True
|
||||
|
||||
# Configure delays for requests for more polite scraping
|
||||
DOWNLOAD_DELAY = 1.5
|
||||
RANDOMIZE_DOWNLOAD_DELAY = True
|
||||
|
||||
# The download delay setting will honor only one of:
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 2
|
||||
CONCURRENT_REQUESTS = 2
|
||||
|
||||
# Enable autothrottle
|
||||
AUTOTHROTTLE_ENABLED = True
|
||||
AUTOTHROTTLE_START_DELAY = 1
|
||||
AUTOTHROTTLE_MAX_DELAY = 3
|
||||
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
|
||||
|
||||
# User agent
|
||||
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
|
||||
# Configure item pipelines
|
||||
ITEM_PIPELINES = {
|
||||
'tibiawiki.pipelines.SQLitePipeline': 300,
|
||||
'tibiawiki.pipelines.CustomImagesPipeline': 400,
|
||||
}
|
||||
|
||||
# Images settings
|
||||
IMAGES_STORE = 'data/images'
|
||||
IMAGES_EXPIRES = 90
|
||||
|
||||
# SQLite database path
|
||||
SQLITE_DB_PATH = 'data/tibiawiki.db'
|
||||
|
||||
# Enable caching
|
||||
HTTPCACHE_ENABLED = False
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL = 'INFO'
|
||||
|
||||
# Feeds exports (opcional - para exportar JSON/CSV)
|
||||
# FEEDS = {
|
||||
# 'data/items.json': {'format': 'json'},
|
||||
# 'data/items.csv': {'format': 'csv'},
|
||||
# }
|
||||
|
||||
# Disable cookies
|
||||
COOKIES_ENABLED = False
|
||||
|
||||
# Request settings
|
||||
DOWNLOAD_TIMEOUT = 30
|
||||
|
||||
# Retry settings
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 3
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
|
||||
# Telnet console
|
||||
TELNETCONSOLE_ENABLED = False
|
||||
|
||||
Reference in new issue
Block a user