<?php

namespace OtomaticAi\Controllers\OneClick;

use Exception;
use OtomaticAi\Models\Persona;
use OtomaticAi\Models\Presets\Preset;
use OtomaticAi\Vendors\Illuminate\Support\Arr;
use OtomaticAi\Vendors\Illuminate\Support\Str;
use OtomaticAi\Controllers\Controller;
use OtomaticAi\Models\Project;
use OtomaticAi\Models\Publication;
use OtomaticAi\Models\Template;
use OtomaticAi\Utils\Language;
use OtomaticAi\Utils\Planning;
use OtomaticAi\Utils\RobotsTxt;
use OtomaticAi\Vendors\Carbon\Carbon;
use OtomaticAi\Vendors\GuzzleHttp\Client;
use OtomaticAi\Vendors\Illuminate\Support\Collection;
use OtomaticAi\Vendors\GuzzleHttp\Pool;
use OtomaticAi\Vendors\GuzzleHttp\Psr7\Request;
use OtomaticAi\Vendors\GuzzleHttp\Psr7\Response;
use OtomaticAi\Vendors\vipnytt\SitemapParser;

class ProjectsController extends Controller
{
    /**
     * Create project
     *
     * @return void
     */
    public function createProject()
    {
        $this->verifyNonce();

        $this->validate([
            "language" => ["required", "string"],
            "models.text" => ["required", "string"],
            "models.image" => ["nullable", "string"],
            "categories" => ["nullable", "array"],
            "profiles" => ["nullable", "array"],
            "project" => ["required", "array"],
        ]);

        $language = Language::find($this->input("language"));
        $models = $this->input("models", ["text" => "gpt-4o-mini", "image" => "flux_1_1_pro"]);
        $project = $this->input("project");
        $categories = $this->input("categories", []);
        $profiles = $this->input("profiles", []);
        $type = Arr::get($project, "type");

        $prj = null;
        switch ($type) {
            case "post":
                $prj = $this->createPostProject($project, $categories, $profiles, $models, $language);
                break;
            case "news":
                $prj = $this->createNewsProject($project, $categories, $profiles, $models, $language);
                break;
            case "url":
                $prj = $this->createUrlProject($project, $categories, $profiles, $models, $language);
                break;
            case "sitemap":
                $prj = $this->createSitemapProject($project, $categories, $profiles, $models, $language);
                break;
            case "rss":
                $prj = $this->createRSSProject($project, $categories, $profiles, $models, $language);
                break;
            case "siloing":
                $prj =  $this->createSiloingProject($project, $profiles, $models, $language);
                break;
        }

        if (!empty($prj)) {
            $this->response([
                "id" => $prj->id,
                "name" => $prj->name,
            ]);
        }

        $this->response(["message" => "The project has not been created."], 422);
    }

    /**
     * Create a post project
     *
     * @param array $project
     * @param array $categories
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createPostProject(array $project, array $categories, array $profiles, array $models, Language $language): ?Project
    {
        // get category
        $category = null;
        if (!empty(Arr::get($project, "category_id"))) {
            $category = Arr::first($categories, function ($c) use ($project) {
                return Arr::get($c, "old_id") === Arr::get($project, "category_id");
            });
        }

        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // get titles
        $titles = [];
        if (!empty(Arr::get($project, "titles"))) {
            $titles = Arr::get($project, "titles");
            $titles = Arr::map($titles, function ($title) use ($project) {
                return [
                    "title" => $title,
                    "meta" => [
                        "keyword" => Arr::get($project, "theme"),
                    ]
                ];
            });
        } else if (!empty(Arr::get($project, "theme"))) {
            $titles = $this->generateTitles($project, $language);
        }

        // make modules
        $modules = $this->makeModules("post", $project, $models, $category, $persona);

        // make planning
        $planning = [
            "enabled" => true,
            "days" => ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("bulk", Arr::get($project, "name"), $titles, $modules, $planning, $persona, $language);
    }

    /**
     * Create a news project
     *
     * @param array $project
     * @param array $categories
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createNewsProject(array $project, array $categories, array $profiles, array $models, Language $language, ?Template $template = null, array $seoOptions = []): ?Project
    {
        // get category
        $category = null;
        if (!empty(Arr::get($project, "category_id"))) {
            $category = Arr::first($categories, function ($c) use ($project) {
                return Arr::get($c, "old_id") === Arr::get($project, "category_id");
            });
        }

        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // make modules
        $modules = $this->makeModules("post", $project, $models, $category, $persona);
        Arr::set($modules, "autopilot", [
            "query" => Arr::get($project, "theme"),
            "source_language" => $language->key,
            "planning" => [
                "per_day" => Arr::get($project, "number_of_posts_to_generate_per_day", 1),
                "start_time" => [
                    "hours" => 6,
                    "minutes" => 30,
                ],
                "end_time" => [
                    "hours" => 21,
                    "minutes" => 0,
                ],
            ],
        ]);

        // make planning
        $planning = [
            "enabled" => false,
            "days" => ["monday", "thursday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("news", Arr::get($project, "name"), [], $modules, $planning, $persona, $language);
    }

    /**
     * Create a url project
     *
     * @param array $project
     * @param array $categories
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createUrlProject(array $project, array $categories, array $profiles, array $models, Language $language): ?Project
    {
        // get category
        $category = null;
        if (!empty(Arr::get($project, "category_id"))) {
            $category = Arr::first($categories, function ($c) use ($project) {
                return Arr::get($c, "old_id") === Arr::get($project, "category_id");
            });
        }

        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // get titles
        $titles = [];
        if (!empty(Arr::get($project, "urls"))) {
            $urls = Arr::get($project, "urls");
            $urls = Arr::map($urls, function ($str) {
                return Str::clean($str);
            });
            $urls = array_values(array_filter($urls, function ($str) {
                return !empty($str);
            }));

            // get urls titles
            $titles = $this->getUrlsTitles($urls);

            // rewrite titles
            if (!empty($titles)) {

                // get the openai preset
                $preset = Preset::findFromAPI("rewrite_title");

                // make payloads
                $payloads = [];
                foreach ($titles as $title) {
                    $payloads[] = [
                        "language" => $language->value,
                        "request" => $title["title"],
                    ];
                }

                // run the pool preset
                $responses = $preset->processPool($payloads);

                // get the response content
                foreach ($responses as $index => $response) {
                    $response = json_decode(Arr::get($response, 'choices.0.message.content'), true, 512, JSON_THROW_ON_ERROR);
                    $item = Arr::get($response, "value");
                    $item = Str::clean($item);
                    if (!empty($item) && isset($titles[$index])) {
                        $titles[$index]["title"] = $item;
                    }
                }

                ksort($titles);
            }
        }

        // make modules
        $modules = $this->makeModules("post", $project, $models, $category, $persona);

        // make planning
        $planning = [
            "enabled" => true,
            "days" => ["monday", "thursday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("bulk", Arr::get($project, "name"), $titles, $modules, $planning, $persona, $language);
    }

    /**
     * Create a sitemap project
     *
     * @param array $project
     * @param array $categories
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createSitemapProject(array $project, array $categories, array $profiles, array $models, Language $language): ?Project
    {
        // get category
        $category = null;
        if (!empty(Arr::get($project, "category_id"))) {
            $category = Arr::first($categories, function ($c) use ($project) {
                return Arr::get($c, "old_id") === Arr::get($project, "category_id");
            });
        }

        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // get titles
        $titles = [];
        if (!empty(Arr::get($project, "sitemap_url"))) {
            // get sitemao url
            $url = Arr::get($project, "sitemap_url");
            $url = Str::clean($url);

            // get urls from sitemap
            $parser = new SitemapParser();
            $parser->parse($url);
            $urls = array_keys($parser->getURLs());

            // take only the number of posts to generate
            $urls = array_slice($urls, 0, Arr::get($project, "number_of_posts_to_generate", 100));

            // get urls titles
            $titles = $this->getUrlsTitles($urls);

            // rewrite titles
            if (!empty($titles)) {

                // get the openai preset
                $preset = Preset::findFromAPI("rewrite_title");

                // make payloads
                $payloads = [];
                foreach ($titles as $title) {
                    $payloads[] = [
                        "language" => $language->value,
                        "request" => $title["title"],
                    ];
                }

                // run the pool preset
                $responses = $preset->processPool($payloads);

                // get the response content
                foreach ($responses as $index => $response) {
                    $response = json_decode(Arr::get($response, 'choices.0.message.content'), true, 512, JSON_THROW_ON_ERROR);
                    $item = Arr::get($response, "value");
                    $item = Str::clean($item);
                    if (!empty($item) && isset($titles[$index])) {
                        $titles[$index]["title"] = $item;
                    }
                }

                ksort($titles);
            }
        }

        // make modules
        $modules = $this->makeModules("post", $project, $models, $category, $persona);

        // make planning
        $planning = [
            "enabled" => true,
            "days" => ["monday", "thursday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("bulk", Arr::get($project, "name"), $titles, $modules, $planning, $persona, $language);
    }

    /**
     * Create a RSS project
     *
     * @param array $project
     * @param array $categories
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createRSSProject(array $project, array $categories, array $profiles, array $models, Language $language): ?Project
    {
        // get category
        $category = null;
        if (!empty(Arr::get($project, "category_id"))) {
            $category = Arr::first($categories, function ($c) use ($project) {
                return Arr::get($c, "old_id") === Arr::get($project, "category_id");
            });
        }

        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // make modules
        $modules = $this->makeModules("post", $project, $models, $category, $persona);
        Arr::set($modules, "autopilot", [
            "query" => Arr::get($project, "rss_url"),
            "source_language" => $language->key,
            "planning" => [
                "per_day" => Arr::get($project, "number_of_posts_to_generate_per_day", 1),
                "start_time" => [
                    "hours" => 6,
                    "minutes" => 30,
                ],
                "end_time" => [
                    "hours" => 21,
                    "minutes" => 0,
                ],
            ],
        ]);

        // make planning
        $planning = [
            "enabled" => false,
            "days" => ["monday", "thursday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("rss", Arr::get($project, "name"), [], $modules, $planning, $persona, $language);
    }

    /**
     * Create a siloing project
     *
     * @param array $project
     * @param array $profiles
     * @param array $models
     * @param Language $language
     * @return Project|null
     */
    private function createSiloingProject(array $project, array $profiles, array $models, Language $language): ?Project
    {
        // get profile
        $persona = null;
        if (!empty(Arr::get($project, "profile_id"))) {
            $persona = Arr::first($profiles, function ($p) use ($project) {
                return Arr::get($p, "old_id") === Arr::get($project, "profile_id");
            });

            if (!empty($persona)) {
                $persona = Persona::find(Arr::get($persona, "id"));
            }
        }

        // get titles
        $titles = $this->generateSiloing($project, $language);

        // make modules
        $modules = $this->makeModules("page", $project, $models, null, $persona);

        // make planning
        $planning = [
            "enabled" => false,
            "days" => ["monday", "thursday", "wednesday", "thursday", "friday", "saturday", "sunday"],
            "per_period" => 1,
            "period" => "day",
            "start_time" => [
                "hours" => 6,
                "minutes" => 30,
            ],
            "end_time" => [
                "hours" => 21,
                "minutes" => 0,
            ],
        ];

        // store the project
        return $this->storeProject("bulk", Arr::get($project, "name"), $titles, $modules, $planning, $persona, $language);
    }

    /**
     * Make the project modules
     *
     * @param string $postType
     * @param array $project
     * @param array $models
     * @param array $category
     * @param Persona $persona
     * @return array
     */
    private function makeModules(string $postType, array $project, array $models, array $category = null, ?Persona $persona = null): array
    {
        $modules = [
            "_version" => Project::$version,
            "models" => [
                "text" => Arr::get($models, "text", "gpt-4o-mini"),
                "image" => Arr::get($models, "image", "dall_e_3"),
            ],
            "content" => [
                "automatic" => [
                    "enabled" => true,
                    "text_length" => "medium",
                    "images" => [
                        "enabled" => empty(Arr::get($models, "image")) ? false : true,
                        "max_images" => 3
                    ],
                ],
            ],
            "wordpress" => [
                "thumbnail" => [
                    "enabled" => empty(Arr::get($models, "image")) ? false : true,
                ],
                "post_type" => $postType,
                "author_id" => $persona ? $persona->user_id : null,
                "categories" => [
                    "automatic" => [
                        "enabled" => false,
                    ],
                    "custom" => $category ? [$category["id"]] : [],
                ],
                "seo_title" => [
                    "enabled" => Arr::get($project, "generate_seo_title", false),
                    "emojis" => [
                        "enabled" => false
                    ]
                ],
                "seo_description" => [
                    "enabled" => Arr::get($project, "generate_seo_description", false),
                ],
            ],
        ];

        return $modules;
    }

    /**
     * Store the project
     *
     * @param string $type
     * @param string $name
     * @param array $titles
     * @param array $modules
     * @param array $planning
     * @param Persona|null $persona
     * @param Language $language
     * @return Project
     */
    private function storeProject(string $type, string $name, array $titles, array $modules, array $planning, ?Persona $persona, Language $language): Project
    {
        // create the project
        $project = new Project([
            "name" => empty($name) ? "New project" : $name,
            "language" => $language->key,
            "type" => $type,
            "enabled" => true,
            "modules" => $modules,
            "planning" => $planning,
        ]);

        if (!empty($persona)) {
            $project->persona()->associate($persona);
        }

        $project->save();

        // flatten requests
        $requests = $this->flattenRequests($titles);

        if (count($requests) > 0) {
            if (!Arr::get($planning, "enabled", false)) {

                foreach ($requests as $key => $request) {
                    $key = strval($request["key"]);
                    $parentKey = isset($request["parent_key"]) ? strval($request["parent_key"]) : '';

                    // create a new publication
                    $publication = new Publication([
                        "title" => $request["title"],
                        "meta" => Arr::get($request, "meta", []),
                        "published_at" => Carbon::now(),
                    ]);

                    // attache the project to publication
                    $publication->project()->associate($project);

                    // get the parent publication and attach it if exist
                    $parentPublication = Arr::get($requests, $parentKey . ".publication");
                    if (!empty($parentPublication)) {
                        $publication->parent_id = $parentPublication;
                    }

                    // save the publication
                    $publication->saveQuietly();
                    $requests[$key]["publication"] = $publication->id;
                }
            } else {
                $schedule = new Planning($requests);
                $schedule
                    ->perPeriod($this->input("per_period", 1))
                    ->period($this->input("period", "day"))
                    ->startTime(Arr::get($planning, "start_time.hours", 6),  Arr::get($planning, "start_time.minutes", 30))
                    ->endTime(Arr::get($planning, "end_time.hours", 21),  Arr::get($planning, "end_time.minutes", 0))
                    ->weekdays(Arr::get($planning, "days", [Planning::MONDAY, Planning::THURSDAY, Planning::WEDNESDAY, Planning::THURSDAY, Planning::FRIDAY]))
                    ->each(function ($request, $date) use ($project, &$requests) {

                        // create a new publication
                        $publication = new Publication([
                            "title" => $request["title"],
                            "meta" => Arr::get($request, "meta", []),
                            "published_at" => $date,
                        ]);

                        // attache the project to publication
                        $publication->project()->associate($project);

                        // get the parent publication and attach it if exist
                        $parentPublication = Arr::get($requests, Arr::get($request, "parent_key") . ".publication");
                        if (!empty($parentPublication)) {
                            $publication->parent_id = $parentPublication;
                        }

                        // save the publication
                        $publication->saveQuietly();
                        $requests[$request["key"]]["publication"] = $publication->id;
                    });
            }

            $project->refreshMetrics();
        }

        return $project;
    }

    /**
     * Flatten the requests
     *
     * @param array $requests
     * @param string|null $parentKey
     * @return array
     */
    private function flattenRequests($requests, $parentKey = null)
    {
        $output = [];
        for ($i = 0; $i < count($requests); $i++) {
            $copy = $requests[$i];
            unset($copy["children"]);
            unset($copy["category_id"]);
            $localKey = $parentKey !== null ? $parentKey . '-' . ($i + 1) : ($i + 1);
            $copy["key"] = $localKey;
            if ($parentKey !== null) {
                $copy["parent_key"] = $parentKey;
            }

            $output[] = $copy;
            if (!empty($requests[$i]["children"])) {
                $output = array_merge($output, $this->flattenRequests($requests[$i]["children"], $localKey));
            }
        }

        return (new Collection($output))->keyBy('key')->toArray();
    }

    /**
     * Generate the titles
     *
     * @param array $project
     * @param Language $language
     * @return array
     */
    private function generateTitles(array $project, Language $language): array
    {
        $titles = [];

        // get the ai preset
        $preset = Preset::findFromAPI("generate_titles");

        // make payload
        $payload = [
            "language" => $language->value,
            "length" => Arr::get($project, "number_of_posts_to_generate", 10),
            "style" => Arr::get($project, "style", "informative"),
            "request" => Arr::get($project, "theme"),
        ];

        try {
            // run the preset
            $response = $preset->process($payload);

            // get the response content
            $response = json_decode(Arr::get($response, 'choices.0.message.content'), true, 512, JSON_THROW_ON_ERROR);
            $titles = Arr::get($response, "values");

            // format the titles
            $titles = Arr::map($titles, function ($title) {
                return Str::clean($title);
            });
            $titles = array_values(array_filter($titles, function ($title) {
                return !empty($title);
            }));
            $titles = Arr::map($titles, function ($title) use ($project) {
                return [
                    "title" => $title,
                    "meta" => [
                        "keyword" => Arr::get($project, "theme"),
                    ]
                ];
            });
        } catch (Exception $e) {
        }

        return $titles;
    }

    /**
     * Get the urls titles
     *
     * @param array $urls
     * @return array
     */
    private function getUrlsTitles(array $urls): array
    {
        $output = [];

        // create the client
        $client = new Client(
            [
                "headers" => [
                    "User-Agent" => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
                ]
            ]
        );

        // make the requests
        $requests = [];
        foreach ($urls as $url) {
            if (!$this->isDisabledByRobotsTxt($url)) {
                $requests[] = new Request(
                    'GET',
                    $url,
                );
            }
        }

        // create the pool
        $pool = new Pool($client, $requests, [
            'concurrency' => 20,
            'fulfilled' => function (Response $response, $index) use (&$output, $urls) {
                $response = $response->getBody()->getContents();
                // h1
                $title = preg_match('/<h1[^>]*>(.*?)<\/h1>/ims', $response, $matches) ? $matches[1] : null;
                if (!empty($urls[$index]) && !empty($title)) {
                    $output[$index] = [
                        "title" => mb_convert_encoding($title, 'UTF-8', 'UTF-8'),
                        "meta" => [
                            "url" => mb_convert_encoding($urls[$index], 'UTF-8', 'UTF-8'),
                        ]
                    ];
                } else {
                    // title
                    $title = preg_match('/<title[^>]*>(.*?)<\/title>/ims', $response, $matches) ? $matches[1] : null;
                    if (!empty($urls[$index]) && !empty($title)) {
                        $output[$index] = [
                            "title" => mb_convert_encoding($title, 'UTF-8', 'UTF-8'),
                            "meta" => [
                                "url" => mb_convert_encoding($urls[$index], 'UTF-8', 'UTF-8'),
                            ]
                        ];
                    }
                }
            },
            'rejected' => function (Exception $reason, $index) {},
        ]);

        // call the pool
        $promise = $pool->promise();
        $promise->wait();

        // sort and get only values
        ksort($output);
        return array_values($output);
    }

    /**
     * Generate the siloing on the main keyword
     *
     * @param array $project
     * @param Language $language
     * @return array
     */
    private function generateSiloing(array $project, Language $language): array
    {
        // get the ai preset
        $preset = Preset::findFromAPI("generate_siloing");

        try {

            // make payload
            $payload = [
                "language" => $language->value,
                "style" => Arr::get($project, "style", "informative"),
                "request" => Arr::get($project, "theme"),
            ];

            // run the preset
            $response = $preset->process($payload);

            // get the response content
            $response = json_decode(Arr::get($response, 'choices.0.message.content'), true, 512, JSON_THROW_ON_ERROR);

            // parse the response
            return $this->parseSiloingItems($response, Arr::get($project, "theme"));
        } catch (Exception $e) {
        }

        return [];
    }

    /**
     * Parse the siloing items
     *
     * @param array $item
     * @param string $keyword
     * @return array
     */
    private function parseSiloingItems(array $item, string $keyword): array
    {
        $output = [];

        if (Arr::isList($item)) {
            foreach ($item as $subItem) {
                $output = array_merge($output, $this->parseSiloingItems($subItem, $keyword));
            }
        } else {
            $title = Arr::get($item, "title");
            $title = Str::clean($title);

            if (!empty($title)) {
                $output[] = [
                    "title" => $title,
                    "meta" => [
                        "keyword" => $keyword
                    ],
                    "children" => empty(Arr::get($item, "children", [])) ? [] : $this->parseSiloingItems(Arr::get($item, "children", []), $keyword)
                ];
            }
        }

        return $output;
    }

    /**
     * Determine if the scrap is disabled by the robots.txt
     *
     * @param string $url
     * @return boolean
     */
    private function isDisabledByRobotsTxt(string $url)
    {
        $robots = RobotsTxt::fromUrl($url);
        return !empty($robots->getRules("otomaticAI"));
    }
}
