Evandro Fonseca Junior
29ba5aece0
Integrates a Scrapy-based TibiaWiki scraper (spider, pipelines, SQLite/Images settings, README). Updates .gitignore to exclude generated data directories (/iconScript/data, /data/images) to keep the repository clean.
75 lines
1.8 KiB
Python
75 lines
1.8 KiB
Python
# Scrapy settings for tibiawiki project
|
|
#
|
|
# For simplicity, this file contains only settings considered important or
|
|
# commonly used. You can find more settings consulting the documentation:
|
|
#
|
|
# https://docs.scrapy.org/en/latest/topics/settings.html
|
|
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
|
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
|
|
|
BOT_NAME = 'tibiawiki'
|
|
|
|
SPIDER_MODULES = ['tibiawiki.spiders']
|
|
NEWSPIDER_MODULE = 'tibiawiki.spiders'
|
|
|
|
|
|
# Obey robots.txt rules
|
|
ROBOTSTXT_OBEY = True
|
|
|
|
# Configure delays for requests for more polite scraping
|
|
DOWNLOAD_DELAY = 1.5
|
|
RANDOMIZE_DOWNLOAD_DELAY = True
|
|
|
|
# The download delay setting will honor only one of:
|
|
CONCURRENT_REQUESTS_PER_DOMAIN = 2
|
|
CONCURRENT_REQUESTS = 2
|
|
|
|
# Enable autothrottle
|
|
AUTOTHROTTLE_ENABLED = True
|
|
AUTOTHROTTLE_START_DELAY = 1
|
|
AUTOTHROTTLE_MAX_DELAY = 3
|
|
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
|
|
|
|
# User agent
|
|
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
|
|
# Configure item pipelines
|
|
ITEM_PIPELINES = {
|
|
'tibiawiki.pipelines.SQLitePipeline': 300,
|
|
'tibiawiki.pipelines.CustomImagesPipeline': 400,
|
|
}
|
|
|
|
# Images settings
|
|
IMAGES_STORE = 'data/images'
|
|
IMAGES_EXPIRES = 90
|
|
|
|
# SQLite database path
|
|
SQLITE_DB_PATH = 'data/tibiawiki.db'
|
|
|
|
# Enable caching
|
|
HTTPCACHE_ENABLED = False
|
|
|
|
# Logging
|
|
LOG_LEVEL = 'INFO'
|
|
|
|
# Feeds exports (opcional - para exportar JSON/CSV)
|
|
# FEEDS = {
|
|
# 'data/items.json': {'format': 'json'},
|
|
# 'data/items.csv': {'format': 'csv'},
|
|
# }
|
|
|
|
# Disable cookies
|
|
COOKIES_ENABLED = False
|
|
|
|
# Request settings
|
|
DOWNLOAD_TIMEOUT = 30
|
|
|
|
# Retry settings
|
|
RETRY_ENABLED = True
|
|
RETRY_TIMES = 3
|
|
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
|
|
|
# Telnet console
|
|
TELNETCONSOLE_ENABLED = False
|
|
|