# Scrapy settings for tibiawiki project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'tibiawiki'

SPIDER_MODULES = ['tibiawiki.spiders']
NEWSPIDER_MODULE = 'tibiawiki.spiders'


# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Configure delays for requests for more polite scraping
DOWNLOAD_DELAY = 1.5
RANDOMIZE_DOWNLOAD_DELAY = True

# The download delay setting will honor only one of:
CONCURRENT_REQUESTS_PER_DOMAIN = 2
CONCURRENT_REQUESTS = 2

# Enable autothrottle
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1
AUTOTHROTTLE_MAX_DELAY = 3
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0

# User agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'

# Configure item pipelines
ITEM_PIPELINES = {
    'tibiawiki.pipelines.SQLitePipeline': 300,
    'tibiawiki.pipelines.CustomImagesPipeline': 400,
}

# Images settings
IMAGES_STORE = 'data/images'
IMAGES_EXPIRES = 90

# SQLite database path
SQLITE_DB_PATH = 'data/tibiawiki.db'

# Enable caching
HTTPCACHE_ENABLED = False

# Logging
LOG_LEVEL = 'INFO'

# Feeds exports (opcional - para exportar JSON/CSV)
# FEEDS = {
#     'data/items.json': {'format': 'json'},
#     'data/items.csv': {'format': 'csv'},
# }

# Disable cookies
COOKIES_ENABLED = False

# Request settings
DOWNLOAD_TIMEOUT = 30

# Retry settings
RETRY_ENABLED = True
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]

# Telnet console
TELNETCONSOLE_ENABLED = False