# Copyright 2022-2025 Laurent Defert
#
#  This file is part of Sosse.
#
# Sosse is free software: you can redistribute it and/or modify it under the terms of the GNU Affero
# General Public License as published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Sosse is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License along with Sosse.
# If not, see <https://www.gnu.org/licenses/>.

# Generated by Django 3.2.12 on 2022-12-25 10:18

import datetime

import django.contrib.postgres.indexes
import django.contrib.postgres.search
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models

import se.models


class Migration(migrations.Migration):
    initial = True

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name="CrawlerStats",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("t", models.DateTimeField()),
                ("doc_count", models.PositiveIntegerField()),
                ("queued_url", models.PositiveIntegerField()),
                ("indexing_speed", models.PositiveIntegerField(blank=True, null=True)),
                (
                    "freq",
                    models.CharField(choices=[("M", "M"), ("D", "D")], max_length=1),
                ),
            ],
        ),
        migrations.CreateModel(
            name="CrawlPolicy",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("url_regex", models.TextField(unique=True)),
                (
                    "condition",
                    models.CharField(
                        choices=[
                            ("always", "Crawl all pages"),
                            ("depth", "Depending on depth"),
                            ("never", "Never crawl"),
                        ],
                        default="always",
                        max_length=6,
                    ),
                ),
                ("mimetype_regex", models.TextField(default="text/.*")),
                (
                    "crawl_depth",
                    models.PositiveIntegerField(
                        default=0,
                        help_text="Level of external links (links that don't match the regex) to recurse into",
                    ),
                ),
                (
                    "keep_params",
                    models.BooleanField(
                        default=True,
                        help_text='When disabled, URL parameters (parameters after "?") are removed from URLs, this can be useful if some parameters are random, change sorting or filtering, ...',
                        verbose_name="Index URL parameters",
                    ),
                ),
                (
                    "default_browse_mode",
                    models.CharField(
                        choices=[
                            ("detect", "Detect"),
                            ("selenium", "Chromium"),
                            ("requests", "Python Requests"),
                        ],
                        default="selenium",
                        help_text="Python Request is faster, but can't execute Javascript and may break pages",
                        max_length=8,
                    ),
                ),
                ("take_screenshots", models.BooleanField(default=True)),
                (
                    "screenshot_format",
                    models.CharField(
                        choices=[("png", "png"), ("jpg", "jpg")],
                        default="jpg",
                        max_length=3,
                    ),
                ),
                (
                    "script",
                    models.TextField(
                        blank=True,
                        default="",
                        help_text="Javascript code to execute after the page is loaded",
                    ),
                ),
                (
                    "store_extern_links",
                    models.BooleanField(default=False, help_text="Store links to non-indexed pages"),
                ),
                (
                    "recrawl_mode",
                    models.CharField(
                        choices=[
                            ("none", "Once"),
                            ("constant", "Constant time"),
                            ("adaptive", "Adaptive"),
                        ],
                        default="adaptive",
                        help_text="Adaptive frequency will increase delay between two crawls when the page stays unchanged",
                        max_length=8,
                        verbose_name="Crawl frequency",
                    ),
                ),
                (
                    "recrawl_dt_min",
                    models.DurationField(
                        blank=True,
                        default=datetime.timedelta(days=1),
                        help_text="Min. time before recrawling a page",
                        null=True,
                    ),
                ),
                (
                    "recrawl_dt_max",
                    models.DurationField(
                        blank=True,
                        default=datetime.timedelta(days=365),
                        help_text="Max. time before recrawling a page",
                        null=True,
                    ),
                ),
                (
                    "hash_mode",
                    models.CharField(
                        choices=[
                            ("raw", "Hash raw content"),
                            ("no_numbers", "Normalize numbers before"),
                        ],
                        default="no_numbers",
                        help_text="Page content hashing method used to detect changes in the content",
                        max_length=10,
                    ),
                ),
                (
                    "auth_login_url_re",
                    models.TextField(
                        blank=True,
                        help_text="A redirection to this URL will trigger authentication",
                        null=True,
                        verbose_name="Login URL",
                    ),
                ),
                (
                    "auth_form_selector",
                    models.TextField(
                        blank=True,
                        help_text="CSS selector pointing to the authentication &lt;form&gt; element",
                        null=True,
                        verbose_name="Form selector",
                    ),
                ),
            ],
            options={
                "verbose_name_plural": "crawl policies",
            },
        ),
        migrations.CreateModel(
            name="DomainSetting",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "browse_mode",
                    models.CharField(
                        choices=[
                            ("detect", "Detect"),
                            ("selenium", "Chromium"),
                            ("requests", "Python Requests"),
                        ],
                        default="detect",
                        max_length=10,
                    ),
                ),
                ("domain", models.TextField(unique=True)),
                (
                    "robots_status",
                    models.CharField(
                        choices=[
                            ("unknown", "Unknown"),
                            ("empty", "Empty"),
                            ("loaded", "Loaded"),
                        ],
                        default="unknown",
                        max_length=10,
                        verbose_name="robots.txt status",
                    ),
                ),
                (
                    "robots_ua_hash",
                    models.CharField(blank=True, default="", max_length=32),
                ),
                (
                    "robots_allow",
                    models.TextField(blank=True, default="", verbose_name="robots.txt allow rules"),
                ),
                (
                    "robots_disallow",
                    models.TextField(blank=True, default="", verbose_name="robots.txt disallow rules"),
                ),
                (
                    "ignore_robots",
                    models.BooleanField(default=False, verbose_name="Ignore robots.txt"),
                ),
            ],
        ),
        migrations.CreateModel(
            name="ExcludedUrl",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("url", models.TextField(unique=True)),
                ("comment", models.TextField(blank=True, null=True)),
            ],
            options={
                "verbose_name": "Excluded URL",
            },
        ),
        migrations.CreateModel(
            name="FavIcon",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("url", models.TextField(unique=True)),
                ("content", models.BinaryField(blank=True, null=True)),
                ("mimetype", models.CharField(blank=True, max_length=64, null=True)),
                ("missing", models.BooleanField(default=True)),
            ],
        ),
        migrations.CreateModel(
            name="SearchEngine",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "short_name",
                    models.CharField(unique=True, blank=True, default="", max_length=32),
                ),
                ("long_name", models.CharField(blank=True, default="", max_length=48)),
                (
                    "description",
                    models.CharField(blank=True, default="", max_length=1024),
                ),
                (
                    "html_template",
                    models.CharField(max_length=2048, validators=[se.models.validate_search_url]),
                ),
                ("shortcut", models.CharField(blank=True, max_length=16)),
            ],
        ),
        migrations.CreateModel(
            name="WorkerStats",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("doc_processed", models.PositiveIntegerField(default=0)),
                ("worker_no", models.IntegerField()),
                ("pid", models.PositiveIntegerField()),
                (
                    "state",
                    models.CharField(
                        choices=[
                            ("idle", "Idle"),
                            ("running", "Running"),
                            ("paused", "Paused"),
                        ],
                        default="idle",
                        max_length=8,
                    ),
                ),
            ],
        ),
        migrations.CreateModel(
            name="SearchHistory",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("query", models.TextField()),
                ("querystring", models.TextField()),
                ("date", models.DateTimeField(auto_now_add=True)),
                (
                    "user",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
            ],
        ),
        migrations.CreateModel(
            name="Document",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "url",
                    models.TextField(unique=True, validators=[se.document.validate_url]),
                ),
                ("normalized_url", models.TextField()),
                ("title", models.TextField()),
                ("normalized_title", models.TextField()),
                ("content", models.TextField()),
                ("normalized_content", models.TextField()),
                ("content_hash", models.TextField(blank=True, null=True)),
                (
                    "vector",
                    django.contrib.postgres.search.SearchVectorField(blank=True, null=True),
                ),
                (
                    "lang_iso_639_1",
                    models.CharField(blank=True, max_length=6, null=True, verbose_name="Language"),
                ),
                ("vector_lang", se.document.RegConfigField(default="simple")),
                ("mimetype", models.CharField(blank=True, max_length=64, null=True)),
                (
                    "robotstxt_rejected",
                    models.BooleanField(default=False, verbose_name="Rejected by robots.txt"),
                ),
                ("redirect_url", models.TextField(blank=True, null=True)),
                ("too_many_redirects", models.BooleanField(default=False)),
                (
                    "screenshot_file",
                    models.CharField(blank=True, max_length=4096, null=True),
                ),
                (
                    "screenshot_count",
                    models.PositiveIntegerField(blank=True, null=True),
                ),
                ("screenshot_size", models.CharField(max_length=16)),
                (
                    "screenshot_format",
                    models.CharField(choices=[("png", "png"), ("jpg", "jpg")], max_length=3),
                ),
                (
                    "crawl_first",
                    models.DateTimeField(blank=True, null=True, verbose_name="Crawled first"),
                ),
                (
                    "crawl_last",
                    models.DateTimeField(blank=True, null=True, verbose_name="Crawled last"),
                ),
                (
                    "crawl_next",
                    models.DateTimeField(blank=True, null=True, verbose_name="Crawl next"),
                ),
                (
                    "crawl_dt",
                    models.DurationField(blank=True, null=True, verbose_name="Crawl DT"),
                ),
                (
                    "crawl_recurse",
                    models.PositiveIntegerField(default=0, verbose_name="Recursion remaining"),
                ),
                ("error", models.TextField(blank=True, default="")),
                ("error_hash", models.TextField(blank=True, default="")),
                ("worker_no", models.PositiveIntegerField(blank=True, null=True)),
                (
                    "favicon",
                    models.ForeignKey(
                        blank=True,
                        null=True,
                        on_delete=django.db.models.deletion.SET_NULL,
                        to="se.favicon",
                    ),
                ),
            ],
        ),
        migrations.CreateModel(
            name="Cookie",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("domain", models.TextField(help_text="Domain name")),
                (
                    "domain_cc",
                    models.TextField(
                        blank=True,
                        help_text="Domain name attribute from the cookie",
                        null=True,
                    ),
                ),
                ("inc_subdomain", models.BooleanField()),
                ("name", models.TextField(blank=True)),
                ("value", models.TextField(blank=True)),
                ("path", models.TextField(default="/")),
                ("expires", models.DateTimeField(blank=True, null=True)),
                ("secure", models.BooleanField()),
                (
                    "same_site",
                    models.CharField(
                        choices=[
                            ("Lax", "Lax"),
                            ("Strict", "Strict"),
                            ("None", "None"),
                        ],
                        default="Lax",
                        max_length=6,
                    ),
                ),
                ("http_only", models.BooleanField(default=False)),
            ],
            options={
                "unique_together": {("domain", "name", "path")},
            },
        ),
        migrations.CreateModel(
            name="AuthField",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "key",
                    models.CharField(max_length=256, verbose_name="<input> name attribute"),
                ),
                ("value", models.CharField(max_length=256)),
                (
                    "crawl_policy",
                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="se.crawlpolicy"),
                ),
            ],
            options={
                "verbose_name": "authentication field",
            },
        ),
        migrations.CreateModel(
            name="Link",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("text", models.TextField(blank=True, null=True)),
                ("pos", models.PositiveIntegerField()),
                ("link_no", models.PositiveIntegerField()),
                ("extern_url", models.TextField(blank=True, null=True)),
                ("screen_pos", models.CharField(blank=True, max_length=64, null=True)),
                (
                    "doc_from",
                    models.ForeignKey(
                        blank=True,
                        null=True,
                        on_delete=django.db.models.deletion.SET_NULL,
                        related_name="links_to",
                        to="se.document",
                    ),
                ),
                (
                    "doc_to",
                    models.ForeignKey(
                        blank=True,
                        null=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="linked_from",
                        to="se.document",
                    ),
                ),
            ],
            options={
                "unique_together": {("doc_from", "link_no")},
            },
        ),
        migrations.AddIndex(
            model_name="document",
            index=django.contrib.postgres.indexes.GinIndex(fields=["vector"], name="se_document_vector_efded7_gin"),
        ),
    ]
