Generation

tibiawikisql.generation

Functions related to generating a database dump from TibiaWiki.

Classes:

Name	Description
Category	Defines the article groups to be fetched.
PostTask	Represents a post-processing task and its category dependencies.

Functions:

Name	Description
img_label	Get the label to show in progress bars when iterating images.
article_label	Get the label to show in progress bar when iterating articles.
constraint	Limit a string to a certain length if exceeded.
progress_bar	Get a progress bar iterator.
fetch_category_entries	Fetch a list of wiki entries in a certain category.
resolve_enabled_categories	Resolve enabled categories including dependency-based auto-skips.
warn_auto_skipped_categories	Emit warnings for categories that were disabled due to dependencies.
run_post_tasks	Run post-processing tasks honoring dependency constraints.
generate	Generate a complete TibiaWiki SQLite database.

Attributes:

Name	Type	Description
CATEGORIES		The categories to fetch and generate objects for.

CATEGORIES module-attribute

CATEGORIES = {
    "achievements": Category(
        "Achievements", AchievementParser, no_images=True
    ),
    "spells": Category(
        "Spells", SpellParser, generate_map=True
    ),
    "items": Category(
        "Objects", ItemParser, generate_map=True
    ),
    "creatures": Category(
        "Creatures", CreatureParser, generate_map=True
    ),
    "books": Category(
        "Book Texts", BookParser, no_images=True
    ),
    "keys": Category(
        "Keys",
        KeyParser,
        no_images=True,
        depends_on=("items",),
    ),
    "npcs": Category("NPCs", NpcParser, generate_map=True),
    "imbuements": Category(
        "Imbuements", ImbuementParser, extension=".png"
    ),
    "quests": Category(
        "Quest Overview Pages", QuestParser, no_images=True
    ),
    "houses": Category(
        "Player-Ownable Buildings",
        HouseParser,
        no_images=True,
    ),
    "charms": Category(
        "Charms", CharmParser, extension=".png"
    ),
    "outfits": Category(
        "Outfits", OutfitParser, no_images=True
    ),
    "worlds": Category(
        "Game Worlds",
        WorldParser,
        no_images=True,
        include_deprecated=True,
    ),
    "mounts": Category("Mounts", MountParser),
    "updates": Category(
        "Updates", UpdateParser, no_images=True
    ),
}

The categories to fetch and generate objects for.

Category

Category(
    name: str | None,
    parser: type[BaseParser],
    *,
    no_images: bool = False,
    extension: str = ".gif",
    include_deprecated: bool = False,
    generate_map: bool = False,
    depends_on: tuple[str, ...] = (),
)

Defines the article groups to be fetched.

Class for internal use only, for easier autocompletion and maintenance.

Parameters:

Name	Type	Description	Default
name	str | None	The name of the TibiaWiki category containing the articles. Doesn't need the Category: prefix.	required
parser	type[BaseParser]	The parser class to use.	required
no_images	bool	Indicate that there is no image extraction from this category's items.	False
extension	str	The filename extension for images.	'.gif'
include_deprecated	bool	Whether to always include deprecated articles from this category.	False
generate_map	bool	Whether to generate a mapping of article names to their article instance for later processing.	False
depends_on	tuple[str, ...]	Category keys required to safely process this category.	()

Source code in tibiawikisql/generation.py

def __init__(
    self,
    name: str | None,
    parser: type[BaseParser],
    *,
    no_images: bool = False,
    extension: str = ".gif",
    include_deprecated: bool = False,
    generate_map: bool = False,
    depends_on: tuple[str, ...] = (),
) -> None:
    """Create a new instance of the class.

    Args:
        name: The name of the TibiaWiki category containing the articles. Doesn't need the `Category:` prefix.
        parser: The parser class to use.
        no_images: Indicate that there is no image extraction from this category's items.
        extension: The filename extension for images.
        include_deprecated: Whether to always include deprecated articles from this category.
        generate_map: Whether to generate a mapping of article names to their article instance for later processing.
        depends_on: Category keys required to safely process this category.

    """
    self.name = name
    self.parser = parser
    self.no_images = no_images
    self.extension = extension
    self.include_deprecated = include_deprecated
    self.generate_map = generate_map
    self.depends_on = depends_on

PostTask dataclass

PostTask(
    name: str,
    callback: Callable[
        [Connection, dict[str, Any], set[str]], None
    ],
    dependencies: tuple[str, ...] = (),
)

Represents a post-processing task and its category dependencies.

img_label

img_label(item: Image | None) -> str

Get the label to show in progress bars when iterating images.

Source code in tibiawikisql/generation.py

def img_label(item: Image | None) -> str:
    """Get the label to show in progress bars when iterating images."""
    if item is None:
        return ""
    return item.clean_name

article_label

article_label(item: Article | None) -> str

Get the label to show in progress bar when iterating articles.

Source code in tibiawikisql/generation.py

def article_label(item: Article | None) -> str:
    """Get the label to show in progress bar when iterating articles."""
    if item is None:
        return ""
    return constraint(item.title, 25)

constraint

constraint(value: str, limit: int) -> str

Limit a string to a certain length if exceeded.

Source code in tibiawikisql/generation.py

def constraint(value: str, limit: int) -> str:
    """Limit a string to a certain length if exceeded."""
    if len(value) <= limit:
        return value
    return value[: limit - 1] + "…"

progress_bar

progress_bar(
    iterable: Iterable[V] | None = None,
    length: int | None = None,
    label: str | None = None,
    item_show_func: Callable[[V | None], str | None]
    | None = None,
    info_sep: str = "  ",
    width: int = 36,
) -> ProgressBar[V]

Get a progress bar iterator.

Source code in tibiawikisql/generation.py

def progress_bar(
    iterable: Iterable[V] | None = None,
    length: int | None = None,
    label: str | None = None,
    item_show_func: Callable[[V | None], str | None] | None = None,
    info_sep: str = "  ",
    width: int = 36,
) -> ProgressBar[V]:
    """Get a progress bar iterator."""
    return click.progressbar(
        iterable,
        length,
        label,
        True,
        True,
        True,
        item_show_func,
        "█",
        "░",
        f"%(label)s {Fore.YELLOW}%(bar)s{Style.RESET_ALL} %(info)s",
        info_sep,
        width,
    )

fetch_category_entries

fetch_category_entries(
    category: str, exclude_titles: set[str] | None = None
) -> list[WikiEntry]

Fetch a list of wiki entries in a certain category.

Source code in tibiawikisql/generation.py

def fetch_category_entries(category: str, exclude_titles: set[str] | None = None) -> list[WikiEntry]:
    """Fetch a list of wiki entries in a certain category."""
    click.echo(f"Fetching articles in {Fore.BLUE}Category:{category}{Style.RESET_ALL}...")
    entries = []
    with timed() as t:
        for entry in wiki_client.get_category_members(category):
            if exclude_titles and entry.title in exclude_titles:
                continue
            if entry.title.startswith("User:") or entry.title.startswith("TibiaWiki:"):
                continue
            entries.append(entry)
    click.echo(f"\t{Fore.GREEN}Found {len(entries):,} articles in {t.elapsed:.2f} seconds.{Style.RESET_ALL}")
    return entries

resolve_enabled_categories

resolve_enabled_categories(
    skip_categories: set[str],
) -> tuple[set[str], dict[str, set[str]]]

Resolve enabled categories including dependency-based auto-skips.

Source code in tibiawikisql/generation.py

def resolve_enabled_categories(skip_categories: set[str]) -> tuple[set[str], dict[str, set[str]]]:
    """Resolve enabled categories including dependency-based auto-skips."""
    enabled_categories = set(CATEGORIES).difference(skip_categories)
    auto_skipped: dict[str, set[str]] = {}
    changed = True
    while changed:
        changed = False
        for key, category in CATEGORIES.items():
            if key not in enabled_categories:
                continue
            missing_dependencies = {dep for dep in category.depends_on if dep not in enabled_categories}
            if not missing_dependencies:
                continue
            enabled_categories.remove(key)
            auto_skipped[key] = missing_dependencies
            changed = True
    return enabled_categories, auto_skipped

warn_auto_skipped_categories

warn_auto_skipped_categories(
    auto_skipped_categories: dict[str, set[str]],
) -> None

Emit warnings for categories that were disabled due to dependencies.

Source code in tibiawikisql/generation.py

def warn_auto_skipped_categories(auto_skipped_categories: dict[str, set[str]]) -> None:
    """Emit warnings for categories that were disabled due to dependencies."""
    for key in CATEGORIES:
        if key not in auto_skipped_categories:
            continue
        dependencies = ", ".join(sorted(auto_skipped_categories[key]))
        click.echo(
            f"{Fore.YELLOW}Skipping category '{key}' because required categories are disabled: "
            f"{dependencies}.{Style.RESET_ALL}",
        )

run_post_tasks

run_post_tasks(
    conn: Connection,
    data_store: dict[str, Any],
    enabled_categories: set[str],
    skip_images: bool,
) -> None

Run post-processing tasks honoring dependency constraints.

Source code in tibiawikisql/generation.py

def run_post_tasks(
    conn: sqlite3.Connection,
    data_store: dict[str, Any],
    enabled_categories: set[str],
    skip_images: bool,
) -> None:
    """Run post-processing tasks honoring dependency constraints."""
    for post_task in POST_TASKS:
        if post_task.name == "images" and skip_images:
            continue
        missing_dependencies = [dep for dep in post_task.dependencies if dep not in enabled_categories]
        if missing_dependencies:
            dependencies = ", ".join(sorted(missing_dependencies))
            click.echo(
                f"{Fore.YELLOW}Skipping task '{post_task.name}' because required categories are disabled: "
                f"{dependencies}.{Style.RESET_ALL}",
            )
            continue
        post_task.callback(conn, data_store, enabled_categories)

generate

generate(
    conn: Connection,
    skip_images: bool = False,
    skip_deprecated: bool = False,
    skip_categories: tuple[str, ...] = (),
) -> None

Generate a complete TibiaWiki SQLite database.

Source code in tibiawikisql/generation.py

def generate(
    conn: sqlite3.Connection,
    skip_images: bool = False,
    skip_deprecated: bool = False,
    skip_categories: tuple[str, ...] = (),
) -> None:
    """Generate a complete TibiaWiki SQLite database."""
    normalized_skip_categories = {category.casefold() for category in skip_categories}
    unknown_categories = normalized_skip_categories - set(CATEGORIES)
    if unknown_categories:
        unknown_str = ", ".join(sorted(unknown_categories))
        msg = f"Unknown categories in skip list: {unknown_str}."
        raise ValueError(msg)

    enabled_categories, auto_skipped_categories = resolve_enabled_categories(normalized_skip_categories)
    warn_auto_skipped_categories(auto_skipped_categories)

    click.echo("Creating schema...")
    schema.create_tables(conn)
    conn.execute("PRAGMA synchronous = OFF")
    data_store: dict[str, Any] = {}

    if skip_deprecated:
        deprecated = {entry.title for entry in fetch_category_entries("Deprecated")}
    else:
        deprecated = set()

    for key, category in CATEGORIES.items():
        if key not in enabled_categories:
            continue
        excluded_titles = deprecated if not category.include_deprecated else None
        data_store[key] = fetch_category_entries(category.name, excluded_titles)

    click.echo("Parsing articles...")
    for key, category in CATEGORIES.items():
        if key not in enabled_categories:
            continue

        titles = [entry.title for entry in data_store[key]]
        parser = category.parser
        if category.generate_map:
            data_store[f"{key}_map"] = {}
        unparsed = []
        generator = wiki_client.get_articles(titles)
        with (
            timed() as t,
            conn,
            progress_bar(generator, len(titles), f"Parsing {key}", item_show_func=article_label) as bar,
        ):
            for article in bar:
                try:
                    entry = parser.from_article(article)
                    entry.insert(conn)
                    if category.generate_map:
                        data_store[f"{key}_map"][entry.title.lower()] = entry.article_id
                except ArticleParsingError:
                    unparsed.append(article.title)
        if unparsed:
            click.echo(f"{Fore.RED}Could not parse {len(unparsed):,} articles.{Style.RESET_ALL}")
            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unparsed)}{Style.RESET_ALL}")
        click.echo(f"\t{Fore.GREEN}Parsed articles in {t.elapsed:.2f} seconds.{Style.RESET_ALL}")

    for position in rashid_positions:
        RashidPositionTable.insert(conn, **position.model_dump())

    run_post_tasks(conn, data_store, enabled_categories, skip_images)

    with conn:
        gen_time = datetime.datetime.now(tz=datetime.timezone.utc)
        schema.DatabaseInfoTable.insert(conn, key="timestamp", value=str(gen_time.timestamp()))
        schema.DatabaseInfoTable.insert(conn, key="generate_time", value=gen_time.isoformat())
        schema.DatabaseInfoTable.insert(conn, key="version", value=__version__)
        schema.DatabaseInfoTable.insert(conn, key="python_version", value=platform.python_version())
        schema.DatabaseInfoTable.insert(conn, key="platform", value=platform.platform())