{"id":5087,"date":"2023-01-30T01:07:44","date_gmt":"2023-01-29T21:37:44","guid":{"rendered":"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/"},"modified":"2023-01-30T01:07:44","modified_gmt":"2023-01-29T21:37:44","slug":"build-seo-site-analyzer-using-python-3g2n","status":"publish","type":"post","link":"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/","title":{"rendered":"\u0633\u0627\u062e\u062a \u062a\u062d\u0644\u06cc\u0644\u06af\u0631 \u0633\u0626\u0648 \u0633\u0627\u06cc\u062a \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 \u067e\u0627\u06cc\u062a\u0648\u0646"},"content":{"rendered":"<div data-article-id=\"1345807\" id=\"article-body\">\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_84 counter-hierarchy ez-toc-counter-rtl ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u0641\u0647\u0631\u0633\u062a \u0645\u0637\u0627\u0644\u0628<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%B2%D9%85%DB%8C%D9%86%D9%87\" >\u0632\u0645\u06cc\u0646\u0647<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%B1%D8%A7%D9%87_%D8%A7%D9%86%D8%AF%D8%A7%D8%B2%DB%8C_%D9%BE%D8%B1%D9%88%DA%98%D9%87\" >\u0631\u0627\u0647 \u0627\u0646\u062f\u0627\u0632\u06cc \u067e\u0631\u0648\u0698\u0647<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%A7%DB%8C%D8%AC%D8%A7%D8%AF_%D9%85%D8%AD%DB%8C%D8%B7_%D9%85%D8%AC%D8%A7%D8%B2%DB%8C\" >\u0627\u06cc\u062c\u0627\u062f \u0645\u062d\u06cc\u0637 \u0645\u062c\u0627\u0632\u06cc<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%DA%A9%D8%AA%D8%A7%D8%A8%D8%AE%D8%A7%D9%86%D9%87_%D9%85%D9%88%D8%B1%D8%AF_%D9%86%DB%8C%D8%A7%D8%B2\" >\u06a9\u062a\u0627\u0628\u062e\u0627\u0646\u0647 \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D9%87%D8%B3%D8%AA%D9%87_%D8%B1%D8%A7_%D8%A7%DB%8C%D8%AC%D8%A7%D8%AF_%DA%A9%D9%86%DB%8C%D8%AF\" >\u0647\u0633\u062a\u0647 \u0631\u0627 \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u06cc\u062f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%AA%D9%86%D8%B8%DB%8C%D9%85_%D9%BE%DB%8C%DA%A9%D8%B1%D8%A8%D9%86%D8%AF%DB%8C_%DA%A9%D8%B1%D9%81%D8%B3\" >\u062a\u0646\u0638\u06cc\u0645 \u067e\u06cc\u06a9\u0631\u0628\u0646\u062f\u06cc \u06a9\u0631\u0641\u0633<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%B3%D8%A7%D8%AE%D8%AA_%D9%85%D8%A7%DA%98%D9%88%D9%84_Scraper\" >\u0633\u0627\u062e\u062a \u0645\u0627\u0698\u0648\u0644 Scraper<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%B3%D8%A7%D8%AE%D8%AA_%D9%86%D9%85%D9%88%D9%86%D9%87_%D8%AE%D8%B2%D9%86%D8%AF%D9%87\" >\u0633\u0627\u062e\u062a \u0646\u0645\u0648\u0646\u0647 \u062e\u0632\u0646\u062f\u0647<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D9%86%D9%85%D9%88%D9%86%D9%87_%D8%A2%D9%86%D8%A7%D9%84%D8%A7%DB%8C%D8%B2%D8%B1_%D8%B3%D8%A7%D8%AE%D8%AA\" >\u0646\u0645\u0648\u0646\u0647 \u0622\u0646\u0627\u0644\u0627\u06cc\u0632\u0631 \u0633\u0627\u062e\u062a<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%A7%D8%B6%D8%A7%D9%81%D9%87_%DA%A9%D8%B1%D8%AF%D9%86_%D9%85%D8%A7%DA%98%D9%88%D9%84_%D9%87%D8%A7%DB%8C_%DA%A9%D9%85%DA%A9%DB%8C\" >\u0627\u0636\u0627\u0641\u0647 \u06a9\u0631\u062f\u0646 \u0645\u0627\u0698\u0648\u0644 \u0647\u0627\u06cc \u06a9\u0645\u06a9\u06cc<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D8%B3%D8%A7%D8%AE%D8%AA_%D8%A8%D8%B1%D9%86%D8%A7%D9%85%D9%87_%D9%87%D8%A7%DB%8C_%D9%88%D8%A8_%D8%A8%D8%A7_%D8%A7%D8%B3%D8%AA%D9%81%D8%A7%D8%AF%D9%87_%D8%A7%D8%B2_Flask\" >\u0633\u0627\u062e\u062a \u0628\u0631\u0646\u0627\u0645\u0647 \u0647\u0627\u06cc \u0648\u0628 \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 Flask<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D9%82%D8%A7%D9%84%D8%A8_%D9%87%D8%A7_%D8%B1%D8%A7_%D8%B7%D8%B1%D8%A7%D8%AD%DB%8C_%DA%A9%D9%86%DB%8C%D8%AF\" >\u0642\u0627\u0644\u0628 \u0647\u0627 \u0631\u0627 \u0637\u0631\u0627\u062d\u06cc \u06a9\u0646\u06cc\u062f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/nabfollower.com\/blog\/build-seo-site-analyzer-using-python-3g2n\/#%D9%86%D8%AA%DB%8C%D8%AC%D9%87\" >\u0646\u062a\u06cc\u062c\u0647<\/a><\/li><\/ul><\/nav><\/div>\n<h3><span class=\"ez-toc-section\" id=\"%D8%B2%D9%85%DB%8C%D9%86%D9%87\"><\/span>\n<p>  \u0632\u0645\u06cc\u0646\u0647<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0647\u0646\u06af\u0627\u0645\u06cc \u06a9\u0647 \u0645\u0646 \u06cc\u06a9 \u0648\u0628 \u0633\u0627\u06cc\u062a \u0631\u0627 \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0645\u06cc \u06a9\u0646\u0645 \u062a\u0627 \u0628\u0631\u062e\u06cc \u0627\u0632 \u06af\u0632\u0627\u0631\u0634 \u0647\u0627\u06cc \u062f\u0627\u062f\u0647 \u0627\u0632 \u06cc\u06a9 \u0647\u062f\u0641 \u0648\u0628 \u0631\u0627 \u062f\u0631\u06cc\u0627\u0641\u062a \u06a9\u0646\u0645\u060c \u0628\u0627\u06cc\u062f \u06cc\u06a9 \u0627\u0633\u06a9\u0631\u0627\u067e\u0631 \u0628\u0644\u0627\u062f\u0631\u0646\u06af \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u0645 \u06a9\u0647 \u0628\u0631\u0627\u06cc \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0648 \u067e\u06cc\u0634 \u0628\u06cc\u0646\u06cc \u06a9\u0644\u0645\u0627\u062a \u06a9\u0644\u06cc\u062f\u06cc \u0648 \u0634\u0645\u0627\u0631\u0634 \u06a9\u0644\u0645\u0647 \u0627\u0632 \u06cc\u06a9 \u0647\u062f\u0641 \u0648\u0628 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0645\u06cc \u0634\u0648\u062f\u060c \u0628\u0646\u0627\u0628\u0631\u0627\u06cc\u0646 \u0686\u0646\u062f \u0627\u0628\u0632\u0627\u0631 \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0628\u0644\u0627\u062f\u0631\u0646\u06af \u0627\u06cc\u062c\u0627\u062f \u06a9\u0631\u062f\u0645. \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 \u0645\u06cc\u06a9\u0631\u0648 \u0641\u0631\u06cc\u0645\u0648\u0631\u06a9 \u067e\u0627\u06cc\u062a\u0648\u0646 \u0648 \u0641\u0644\u0627\u0633\u06a9<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%D8%B1%D8%A7%D9%87_%D8%A7%D9%86%D8%AF%D8%A7%D8%B2%DB%8C_%D9%BE%D8%B1%D9%88%DA%98%D9%87\"><\/span>\n<p>  \u0631\u0627\u0647 \u0627\u0646\u062f\u0627\u0632\u06cc \u067e\u0631\u0648\u0698\u0647<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u0627\u0628\u062a\u062f\u0627 \u0645\u062f\u06cc\u0631\u06cc\u062a \u0628\u0633\u062a\u0647 \u0634\u0639\u0631 \u0631\u0627 \u0628\u0631\u0627\u06cc \u0645\u062f\u06cc\u0631\u06cc\u062a \u0628\u0633\u062a\u0647 \u067e\u0627\u06cc\u062a\u0648\u0646 \u0646\u0635\u0628 \u06a9\u0646\u06cc\u062f\u060c \u0627\u062c\u0627\u0632\u0647 \u062f\u0647\u06cc\u062f \u0634\u0631\u0648\u0639 \u0628\u0647 \u0631\u0627\u0647 \u0627\u0646\u062f\u0627\u0632\u06cc \u067e\u0631\u0648\u0698\u0647 \u06a9\u0646\u06cc\u0645<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%D8%A7%DB%8C%D8%AC%D8%A7%D8%AF_%D9%85%D8%AD%DB%8C%D8%B7_%D9%85%D8%AC%D8%A7%D8%B2%DB%8C\"><\/span>\n<p>  \u0627\u06cc\u062c\u0627\u062f \u0645\u062d\u06cc\u0637 \u0645\u062c\u0627\u0632\u06cc<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0627\u06cc\u062c\u0627\u062f \u0645\u062d\u06cc\u0637 \u0645\u062c\u0627\u0632\u06cc<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight shell\"><code> python <span class=\"nt\">-m<\/span> venv venv\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight shell\"><code> venv<span class=\"se\">\\S<\/span>cripts<span class=\"se\">\\a<\/span>ctivate <span class=\"c\"># on windows<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight shell\"><code><span class=\"nb\">source <\/span>venv\/bin\/activate <span class=\"c\"># on mac or linux<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<p><strong>\u06a9\u062a\u0627\u0628\u062e\u0627\u0646\u0647 \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632 \u0631\u0627 \u0646\u0635\u0628 \u06a9\u0646\u06cc\u062f<\/strong><\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"n\">pip<\/span> <span class=\"n\">install<\/span> <span class=\"n\">poetry<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h3><span class=\"ez-toc-section\" id=\"%DA%A9%D8%AA%D8%A7%D8%A8%D8%AE%D8%A7%D9%86%D9%87_%D9%85%D9%88%D8%B1%D8%AF_%D9%86%DB%8C%D8%A7%D8%B2\"><\/span>\n<p>  \u06a9\u062a\u0627\u0628\u062e\u0627\u0646\u0647 \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u06a9\u062a\u0627\u0628\u062e\u0627\u0646\u0647 \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632 \u0632\u06cc\u0631 \u0631\u0627 \u0646\u0635\u0628 \u06a9\u0646\u06cc\u062f<\/p>\n<ul>\n<li>\u062f\u0631\u062e\u0648\u0627\u0633\u062a \u0647\u0627<\/li>\n<li>pyseoanalyzer<\/li>\n<li>\u0633\u0648\u067e \u0632\u06cc\u0628\u0627 4<\/li>\n<li>\u0641\u0644\u0627\u0633\u06a9<\/li>\n<li>\u06a9\u0631\u0641\u0633<\/li>\n<li>\u0628\u06a9\u0634 \u06a9\u0646\u0627\u0631<\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"%D9%87%D8%B3%D8%AA%D9%87_%D8%B1%D8%A7_%D8%A7%DB%8C%D8%AC%D8%A7%D8%AF_%DA%A9%D9%86%DB%8C%D8%AF\"><\/span>\n<p>  \u0647\u0633\u062a\u0647 \u0631\u0627 \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u06cc\u062f<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0627\u0628\u062a\u062f\u0627 \u0647\u0633\u062a\u0647 \u0628\u0646\u06cc\u0627\u062f \u067e\u0631\u0648\u0698\u0647 \u0631\u0627 \u0628\u0647 \u0627\u06cc\u0646 \u0635\u0648\u0631\u062a \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u06cc\u062f<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/core\/factory.py\n<\/span><span class=\"kn\">import<\/span> <span class=\"nn\">os<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">flask<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Flask<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">core<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">init_celery<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">apps.home.views<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">home_bp<\/span>\n\n<span class=\"k\">def<\/span> <span class=\"nf\">create_app<\/span><span class=\"p\">(<\/span><span class=\"o\">**<\/span><span class=\"n\">kwargs<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">Flask<\/span><span class=\"p\">:<\/span>\n    <span class=\"n\">app<\/span><span class=\"p\">:<\/span> <span class=\"n\">Flask<\/span> <span class=\"o\">=<\/span> <span class=\"n\">Flask<\/span><span class=\"p\">(<\/span><span class=\"n\">__name__<\/span><span class=\"p\">)<\/span>\n    <span class=\"c1\"># environtment\n<\/span>    <span class=\"n\">env<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"MODE\"<\/span><span class=\"p\">)<\/span>\n    <span class=\"k\">if<\/span> <span class=\"n\">env<\/span> <span class=\"o\">==<\/span> <span class=\"s\">\"development\"<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"Run In Development Mode\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">config<\/span><span class=\"p\">.<\/span><span class=\"n\">from_object<\/span><span class=\"p\">(<\/span><span class=\"s\">\"core.config.DevConfig\"<\/span><span class=\"p\">)<\/span>\n    <span class=\"k\">else<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"Run In Production Mode\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">config<\/span><span class=\"p\">.<\/span><span class=\"n\">from_object<\/span><span class=\"p\">(<\/span><span class=\"s\">\"core.config.ProdConfig\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">if<\/span> <span class=\"n\">kwargs<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"celery\"<\/span><span class=\"p\">):<\/span>\n            <span class=\"n\">init_celery<\/span><span class=\"p\">(<\/span><span class=\"n\">app<\/span><span class=\"o\">=<\/span><span class=\"n\">app<\/span><span class=\"p\">,<\/span> <span class=\"n\">celery<\/span><span class=\"o\">=<\/span><span class=\"n\">kwargs<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"celery\"<\/span><span class=\"p\">))<\/span>\n\n    <span class=\"c1\"># initiate blueprint\n<\/span>    <span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">register_blueprint<\/span><span class=\"p\">(<\/span><span class=\"n\">home_bp<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"k\">return<\/span> <span class=\"n\">app<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h3><span class=\"ez-toc-section\" id=\"%D8%AA%D9%86%D8%B8%DB%8C%D9%85_%D9%BE%DB%8C%DA%A9%D8%B1%D8%A8%D9%86%D8%AF%DB%8C_%DA%A9%D8%B1%D9%81%D8%B3\"><\/span>\n<p>  \u062a\u0646\u0638\u06cc\u0645 \u067e\u06cc\u06a9\u0631\u0628\u0646\u062f\u06cc \u06a9\u0631\u0641\u0633<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0645\u0631\u062d\u0644\u0647 \u0628\u0639\u062f\u06cc \u062a\u0646\u0638\u06cc\u0645 \u067e\u06cc\u06a9\u0631\u0628\u0646\u062f\u06cc \u06a9\u0631\u0641\u0633 \u0627\u0633\u062a<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/core\/__init__.py\n<\/span><span class=\"kn\">import<\/span> <span class=\"nn\">os<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">celery<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Celery<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">flask<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Flask<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Any<\/span>\n\n<span class=\"k\">def<\/span> <span class=\"nf\">celery_app<\/span><span class=\"p\">(<\/span><span class=\"n\">app_name<\/span><span class=\"o\">=<\/span><span class=\"n\">__name__<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">Celery<\/span><span class=\"p\">:<\/span>\n    <span class=\"s\">\"\"\"create celery object to make a module of celery task\n    Args:\n        app (None): celery app object\n    Returns:\n        Celery: return configured Celery Objects\n    \"\"\"<\/span>\n    <span class=\"k\">if<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"MODE\"<\/span><span class=\"p\">)<\/span> <span class=\"o\">==<\/span> <span class=\"s\">\"development\"<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"Broker development\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">redis_broker<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_BROKER_URL_DEV\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">redis_backend<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_RESULT_BACKEND_DEV\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">return<\/span> <span class=\"n\">Celery<\/span><span class=\"p\">(<\/span><span class=\"n\">app_name<\/span><span class=\"p\">,<\/span> <span class=\"n\">backend<\/span><span class=\"o\">=<\/span><span class=\"n\">redis_backend<\/span><span class=\"p\">,<\/span> <span class=\"n\">redis_broker<\/span><span class=\"o\">=<\/span><span class=\"n\">redis_broker<\/span><span class=\"p\">)<\/span>\n    <span class=\"k\">else<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"Broker Production\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">redis_broker<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_BROKER_URL\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">redis_backend<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_RESULT_BACKEND\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">return<\/span> <span class=\"n\">Celery<\/span><span class=\"p\">(<\/span><span class=\"n\">app_name<\/span><span class=\"p\">,<\/span> <span class=\"n\">backend<\/span><span class=\"o\">=<\/span><span class=\"n\">redis_backend<\/span><span class=\"p\">,<\/span> <span class=\"n\">redis_broker<\/span><span class=\"o\">=<\/span><span class=\"n\">redis_broker<\/span><span class=\"p\">)<\/span>\n\n\n<span class=\"k\">def<\/span> <span class=\"nf\">init_celery<\/span><span class=\"p\">(<\/span><span class=\"n\">celery<\/span><span class=\"p\">:<\/span> <span class=\"n\">Celery<\/span><span class=\"p\">,<\/span> <span class=\"n\">app<\/span><span class=\"p\">:<\/span> <span class=\"n\">Flask<\/span><span class=\"p\">):<\/span>\n    <span class=\"s\">\"\"\"Adding Flask to Celery Support\n    Args:\n        app (Flask): Flask Object\n    Returns:\n        Celery: Celery Object\n    \"\"\"<\/span>\n\n    <span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">config<\/span><span class=\"p\">.<\/span><span class=\"n\">update<\/span><span class=\"p\">({<\/span>\n        <span class=\"s\">'broker_url'<\/span><span class=\"p\">:<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_BROKER_URL\"<\/span><span class=\"p\">),<\/span>\n        <span class=\"s\">\"result_backend\"<\/span><span class=\"p\">:<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_RESULT_BACKEND\"<\/span><span class=\"p\">)<\/span>\n    <span class=\"p\">})<\/span>\n\n\n    <span class=\"n\">celery<\/span><span class=\"p\">.<\/span><span class=\"n\">conf<\/span><span class=\"p\">.<\/span><span class=\"n\">update<\/span><span class=\"p\">(<\/span><span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">config<\/span><span class=\"p\">)<\/span>\n    <span class=\"n\">TaskBase<\/span> <span class=\"o\">=<\/span> <span class=\"n\">celery<\/span><span class=\"p\">.<\/span><span class=\"n\">Task<\/span>\n    <span class=\"k\">class<\/span> <span class=\"nc\">ContextTask<\/span><span class=\"p\">(<\/span><span class=\"n\">TaskBase<\/span><span class=\"p\">):<\/span>\n        <span class=\"n\">abstract<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">True<\/span>\n        <span class=\"k\">def<\/span> <span class=\"nf\">__call__<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"o\">*<\/span><span class=\"n\">args<\/span><span class=\"p\">:<\/span> <span class=\"n\">Any<\/span><span class=\"p\">,<\/span> <span class=\"o\">**<\/span><span class=\"n\">kwargs<\/span><span class=\"p\">:<\/span> <span class=\"n\">Any<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">Any<\/span><span class=\"p\">:<\/span>\n            <span class=\"k\">with<\/span> <span class=\"n\">app<\/span><span class=\"p\">.<\/span><span class=\"n\">app_context<\/span><span class=\"p\">():<\/span>\n                <span class=\"k\">return<\/span> <span class=\"n\">TaskBase<\/span><span class=\"p\">.<\/span><span class=\"n\">__call__<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"o\">*<\/span><span class=\"n\">args<\/span><span class=\"p\">,<\/span> <span class=\"o\">**<\/span><span class=\"n\">kwargs<\/span><span class=\"p\">)<\/span>\n    <span class=\"n\">celery<\/span><span class=\"p\">.<\/span><span class=\"n\">Task<\/span> <span class=\"o\">=<\/span> <span class=\"n\">ContextTask<\/span>\n    <span class=\"k\">return<\/span> <span class=\"n\">celery<\/span>\n\n\n<span class=\"n\">celery_ext<\/span> <span class=\"o\">=<\/span> <span class=\"n\">celery_app<\/span><span class=\"p\">()<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<p>\u067e\u0633 \u0627\u0632 \u0627\u06cc\u062c\u0627\u062f \u067e\u0633\u0648\u0646\u062f \u06a9\u0631\u0641\u0633\u060c \u0628\u06cc\u0627\u06cc\u06cc\u062f \u0622\u0646 \u0631\u0627 \u0628\u0633\u0627\u0632\u06cc\u0645 <code>config<\/code> \u0628\u0631\u0627\u06cc \u067e\u06cc\u06a9\u0631\u0628\u0646\u062f\u06cc \u067e\u0631\u0648\u0698\u0647<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/core\/config.py \n<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">os<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Literal<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">pathlib<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Path<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">dotenv<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">dotenv_values<\/span>\n\n<span class=\"n\">dotenv_path<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">path<\/span><span class=\"p\">.<\/span><span class=\"n\">join<\/span><span class=\"p\">(<\/span><span class=\"n\">Path<\/span><span class=\"p\">(<\/span><span class=\"n\">__file__<\/span><span class=\"p\">).<\/span><span class=\"n\">resolve<\/span><span class=\"p\">().<\/span><span class=\"n\">parent<\/span><span class=\"p\">.<\/span><span class=\"n\">parent<\/span><span class=\"p\">.<\/span><span class=\"n\">parent<\/span><span class=\"p\">,<\/span> <span class=\"s\">'.env'<\/span><span class=\"p\">)<\/span>\n\n<span class=\"n\">tes<\/span> <span class=\"o\">=<\/span> <span class=\"n\">dotenv_values<\/span><span class=\"p\">(<\/span><span class=\"n\">dotenv_path<\/span><span class=\"p\">)<\/span>\n\n<span class=\"k\">class<\/span> <span class=\"nc\">BaseConfig<\/span><span class=\"p\">:<\/span>\n    <span class=\"n\">SECRET_KEY<\/span><span class=\"p\">:<\/span> <span class=\"n\">Literal<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"SECRET_KEY\"<\/span><span class=\"p\">)<\/span>\n\n\n<span class=\"k\">class<\/span> <span class=\"nc\">DevConfig<\/span><span class=\"p\">(<\/span><span class=\"n\">BaseConfig<\/span><span class=\"p\">):<\/span>\n    <span class=\"n\">DEBUG<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">True<\/span>\n    <span class=\"n\">FLASK_DEBUG<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span><span class=\"bp\">True<\/span>\n\n<span class=\"k\">class<\/span> <span class=\"nc\">ProdConfig<\/span><span class=\"p\">(<\/span><span class=\"n\">BaseConfig<\/span><span class=\"p\">):<\/span>\n    <span class=\"n\">DEBUG<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">False<\/span>\n    <span class=\"n\">broker_url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_BROKER_URL\"<\/span><span class=\"p\">)<\/span>\n    <span class=\"n\">RESULT_BACKEND<\/span> <span class=\"o\">=<\/span> <span class=\"n\">os<\/span><span class=\"p\">.<\/span><span class=\"n\">environ<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"CELERY_RESULT_BACKEND\"<\/span><span class=\"p\">)<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h2><span class=\"ez-toc-section\" id=\"%D8%B3%D8%A7%D8%AE%D8%AA_%D9%85%D8%A7%DA%98%D9%88%D9%84_Scraper\"><\/span>\n<p>  \u0633\u0627\u062e\u062a \u0645\u0627\u0698\u0648\u0644 Scraper<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u062f\u0631 \u0627\u06cc\u0646 \u0645\u0648\u0631\u062f \u0645\u0627 \u0628\u0627\u06cc\u062f \u0645\u0627\u0698\u0648\u0644 scraper \u0631\u0627 \u062f\u0648\u0628\u0627\u0631\u0647 \u0628\u0633\u0627\u0632\u06cc\u0645 \u062a\u0627 \u06cc\u06a9 \u0646\u062a\u0627\u06cc\u062c \u0633\u0641\u0627\u0631\u0634\u06cc \u0628\u0631\u0627\u06cc API \u062e\u0648\u062f \u0628\u0633\u0627\u0632\u06cc\u0645<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%D8%B3%D8%A7%D8%AE%D8%AA_%D9%86%D9%85%D9%88%D9%86%D9%87_%D8%AE%D8%B2%D9%86%D8%AF%D9%87\"><\/span>\n<p>  \u0633\u0627\u062e\u062a \u0646\u0645\u0648\u0646\u0647 \u062e\u0632\u0646\u062f\u0647<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">seoanalyzer.website<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Website<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">xml.dom<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">minidom<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">modules.scrape<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">PageCrawler<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">modules.helper<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">http<\/span>\n\n<span class=\"k\">class<\/span> <span class=\"nc\">WebCrawler<\/span><span class=\"p\">(<\/span><span class=\"n\">Website<\/span><span class=\"p\">):<\/span>\n    <span class=\"k\">def<\/span> <span class=\"nf\">__init__<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">base_url<\/span><span class=\"p\">,<\/span> <span class=\"n\">sitemap<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_headings<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">,<\/span> <span class=\"n\">follow_links<\/span><span class=\"p\">,<\/span> <span class=\"n\">scrape_img<\/span><span class=\"p\">):<\/span>\n        <span class=\"nb\">super<\/span><span class=\"p\">().<\/span><span class=\"n\">__init__<\/span><span class=\"p\">(<\/span><span class=\"n\">base_url<\/span><span class=\"p\">,<\/span> <span class=\"n\">sitemap<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_headings<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">,<\/span> <span class=\"n\">follow_links<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span> <span class=\"o\">=<\/span> <span class=\"n\">scrape_img<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">crawl<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">):<\/span>\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">sitemap<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">page<\/span> <span class=\"o\">=<\/span> <span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">sitemap<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">sitemap<\/span><span class=\"p\">.<\/span><span class=\"n\">endswith<\/span><span class=\"p\">(<\/span><span class=\"s\">'xml'<\/span><span class=\"p\">):<\/span>\n                <span class=\"n\">xmldoc<\/span> <span class=\"o\">=<\/span> <span class=\"n\">minidom<\/span><span class=\"p\">.<\/span><span class=\"n\">parseString<\/span><span class=\"p\">(<\/span><span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">data<\/span><span class=\"p\">.<\/span><span class=\"n\">decode<\/span><span class=\"p\">(<\/span><span class=\"s\">'utf-8'<\/span><span class=\"p\">))<\/span>\n                <span class=\"n\">sitemap_urls<\/span> <span class=\"o\">=<\/span> <span class=\"n\">xmldoc<\/span><span class=\"p\">.<\/span><span class=\"n\">getElementsByTagName<\/span><span class=\"p\">(<\/span><span class=\"s\">'loc'<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">for<\/span> <span class=\"n\">url<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">sitemap_urls<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">page_queue<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">get_text_from_xml<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">.<\/span><span class=\"n\">childNodes<\/span><span class=\"p\">))<\/span>\n            <span class=\"k\">elif<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">sitemap<\/span><span class=\"p\">.<\/span><span class=\"n\">endswith<\/span><span class=\"p\">(<\/span><span class=\"s\">'txt'<\/span><span class=\"p\">):<\/span>\n                <span class=\"n\">sitemap_urls<\/span> <span class=\"o\">=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">data<\/span><span class=\"p\">.<\/span><span class=\"n\">decode<\/span><span class=\"p\">(<\/span><span class=\"s\">'utf-8'<\/span><span class=\"p\">).<\/span><span class=\"n\">split<\/span><span class=\"p\">(<\/span><span class=\"s\">'<\/span><span class=\"se\">\\n<\/span><span class=\"s\">'<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">for<\/span> <span class=\"n\">url<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">sitemap_urls<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">page_queue<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n\n\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">page_queue<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">base_url<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">for<\/span> <span class=\"n\">url<\/span> <span class=\"ow\">in<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">page_queue<\/span><span class=\"p\">:<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"n\">url<\/span> <span class=\"ow\">in<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">crawled_urls<\/span><span class=\"p\">:<\/span>\n                <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"Crawled URL: {}\"<\/span><span class=\"p\">.<\/span><span class=\"nb\">format<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">))<\/span>\n                <span class=\"k\">continue<\/span>\n            <span class=\"k\">else<\/span><span class=\"p\">:<\/span>\n                <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"s\">\"process URL to crawl: {}\"<\/span><span class=\"p\">.<\/span><span class=\"nb\">format<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">))<\/span>\n                <span class=\"n\">page<\/span> <span class=\"o\">=<\/span> <span class=\"n\">PageCrawler<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"o\">=<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">base_url<\/span><span class=\"o\">=<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">base_url<\/span><span class=\"p\">,<\/span>\n                            <span class=\"n\">analyze_headings<\/span><span class=\"o\">=<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">,<\/span>\n                            <span class=\"n\">analyze_extra_tags<\/span><span class=\"o\">=<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">,<\/span> <span class=\"n\">scrape_img<\/span><span class=\"o\">=<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">)<\/span>\n\n                <span class=\"k\">if<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">parsed_url<\/span><span class=\"p\">.<\/span><span class=\"n\">netloc<\/span> <span class=\"o\">!=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">base_domain<\/span><span class=\"p\">.<\/span><span class=\"n\">netloc<\/span><span class=\"p\">:<\/span>\n                    <span class=\"k\">continue<\/span>\n\n                <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze<\/span><span class=\"p\">()<\/span>\n\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hashes<\/span><span class=\"p\">[<\/span><span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hash<\/span><span class=\"p\">].<\/span><span class=\"n\">add<\/span><span class=\"p\">(<\/span><span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n\n                <span class=\"k\">for<\/span> <span class=\"n\">w<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">wordcount<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">wordcount<\/span><span class=\"p\">[<\/span><span class=\"n\">w<\/span><span class=\"p\">]<\/span> <span class=\"o\">+=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">wordcount<\/span><span class=\"p\">[<\/span><span class=\"n\">w<\/span><span class=\"p\">]<\/span>\n\n                <span class=\"k\">for<\/span> <span class=\"n\">b<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">bigrams<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">bigrams<\/span><span class=\"p\">[<\/span><span class=\"n\">b<\/span><span class=\"p\">]<\/span> <span class=\"o\">+=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">bigrams<\/span><span class=\"p\">[<\/span><span class=\"n\">b<\/span><span class=\"p\">]<\/span>\n\n                <span class=\"k\">for<\/span> <span class=\"n\">t<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">trigrams<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">trigrams<\/span><span class=\"p\">[<\/span><span class=\"n\">t<\/span><span class=\"p\">]<\/span> <span class=\"o\">+=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">trigrams<\/span><span class=\"p\">[<\/span><span class=\"n\">t<\/span><span class=\"p\">]<\/span>\n\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">page_queue<\/span><span class=\"p\">.<\/span><span class=\"n\">extend<\/span><span class=\"p\">(<\/span><span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">links<\/span><span class=\"p\">)<\/span>\n\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">crawled_pages<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">page<\/span><span class=\"p\">)<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">crawled_urls<\/span><span class=\"p\">.<\/span><span class=\"n\">add<\/span><span class=\"p\">(<\/span><span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n\n                <span class=\"k\">if<\/span> <span class=\"ow\">not<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">follow_links<\/span><span class=\"p\">:<\/span>\n                    <span class=\"k\">break<\/span>\n\n                <span class=\"k\">if<\/span> <span class=\"ow\">not<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span>\n                    <span class=\"k\">break<\/span> \n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<p>\u0648 \u06cc\u06a9 \u062e\u0632\u0646\u062f\u0647 \u0635\u0641\u062d\u0647 \u0628\u0631\u0627\u06cc \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0633\u0627\u06cc\u062a \u062f\u0631 \u0647\u0631 \u0635\u0641\u062d\u0647 \u0628\u0633\u0627\u0632\u06cc\u062f<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/modules\/scrape.py\n<\/span>\n<span class=\"c1\"># file to process page\n<\/span><span class=\"kn\">import<\/span> <span class=\"nn\">re<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">hashlib<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">requests<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">seoanalyzer.page<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Page<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">bs4<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">BeautifulSoup<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib3.exceptions<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">HTTPError<\/span><span class=\"p\">,<\/span> <span class=\"n\">MaxRetryError<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib.parse<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">urlsplit<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Any<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">modules.helper<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">http<\/span>\n\n\n<span class=\"k\">class<\/span> <span class=\"nc\">PageCrawler<\/span><span class=\"p\">(<\/span><span class=\"n\">Page<\/span><span class=\"p\">):<\/span>\n    <span class=\"k\">def<\/span> <span class=\"nf\">__init__<\/span><span class=\"p\">(<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">,<\/span>\n        <span class=\"n\">url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">,<\/span>\n        <span class=\"n\">base_url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">,<\/span>\n        <span class=\"n\">analyze_headings<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">False<\/span><span class=\"p\">,<\/span>\n        <span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">False<\/span><span class=\"p\">,<\/span>\n        <span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">False<\/span><span class=\"p\">,<\/span>\n    <span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"bp\">None<\/span><span class=\"p\">:<\/span>\n        <span class=\"nb\">super<\/span><span class=\"p\">().<\/span><span class=\"n\">__init__<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">base_url<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_headings<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"n\">scrape_img<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"n\">analyze_headings<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">:<\/span> <span class=\"nb\">bool<\/span> <span class=\"o\">=<\/span> <span class=\"n\">analyze_extra_tags<\/span>\n\n        <span class=\"c1\"># adding broken and internal link\n<\/span>        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">internal_links<\/span><span class=\"p\">:<\/span> <span class=\"nb\">list<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">broken_link<\/span><span class=\"p\">:<\/span> <span class=\"nb\">list<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">external_link<\/span><span class=\"p\">:<\/span> <span class=\"nb\">list<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">images<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">headings<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{}<\/span>\n        <span class=\"k\">if<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">additional_info<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{}<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">scrape_external_link<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">bs<\/span><span class=\"p\">:<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">):<\/span>\n        <span class=\"n\">external_link<\/span> <span class=\"o\">=<\/span> <span class=\"n\">bs<\/span><span class=\"p\">.<\/span><span class=\"n\">find_all<\/span><span class=\"p\">(<\/span><span class=\"s\">\"a\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">hostname<\/span> <span class=\"o\">=<\/span> <span class=\"n\">urlsplit<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">).<\/span><span class=\"n\">hostname<\/span>\n        <span class=\"k\">for<\/span> <span class=\"n\">link<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">external_link<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">valid_link<\/span> <span class=\"o\">=<\/span> <span class=\"n\">link<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"href\"<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">if<\/span> <span class=\"n\">valid_link<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"http:\/\/\"<\/span><span class=\"p\">)<\/span> <span class=\"ow\">or<\/span> <span class=\"n\">valid_link<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"https:\/\/\"<\/span><span class=\"p\">):<\/span>\n                <span class=\"k\">if<\/span> <span class=\"n\">urlsplit<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_link<\/span><span class=\"p\">).<\/span><span class=\"n\">hostname<\/span> <span class=\"o\">!=<\/span> <span class=\"n\">hostname<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">external_link<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_link<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">external_link<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">scrape_broken_link<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">bs<\/span><span class=\"p\">:<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"nb\">list<\/span><span class=\"p\">:<\/span>\n        <span class=\"n\">broken_link<\/span> <span class=\"o\">=<\/span> <span class=\"n\">bs<\/span><span class=\"p\">.<\/span><span class=\"n\">find_all<\/span><span class=\"p\">(<\/span><span class=\"s\">\"a\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">for<\/span> <span class=\"n\">broke<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">broken_link<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">valid_url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span> <span class=\"o\">=<\/span> <span class=\"n\">broke<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"href\"<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">if<\/span> <span class=\"n\">valid_url<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"http:\/\/\"<\/span><span class=\"p\">)<\/span> <span class=\"ow\">or<\/span> <span class=\"n\">valid_url<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"https:\/\/\"<\/span><span class=\"p\">):<\/span>\n                <span class=\"k\">try<\/span><span class=\"p\">:<\/span>\n                    <span class=\"n\">broken<\/span> <span class=\"o\">=<\/span> <span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_url<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">except<\/span> <span class=\"n\">MaxRetryError<\/span><span class=\"p\">:<\/span>\n                    <span class=\"n\">broken<\/span> <span class=\"o\">=<\/span> <span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">get_without_redirect<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_url<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">try<\/span><span class=\"p\">:<\/span>\n                    <span class=\"k\">if<\/span> <span class=\"n\">broken<\/span><span class=\"p\">.<\/span><span class=\"n\">status<\/span> <span class=\"o\">==<\/span> <span class=\"mi\">404<\/span><span class=\"p\">:<\/span>\n                        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">broken_link<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_url<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">except<\/span><span class=\"p\">:<\/span>\n                    <span class=\"k\">if<\/span> <span class=\"n\">broken<\/span><span class=\"p\">.<\/span><span class=\"n\">status_code<\/span> <span class=\"o\">==<\/span> <span class=\"mi\">404<\/span><span class=\"p\">:<\/span>\n                        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">broken_link<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_url<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">else<\/span><span class=\"p\">:<\/span>\n                <span class=\"k\">continue<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">broken_link<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">scrape_internal_link<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">bs<\/span><span class=\"p\">:<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"nb\">list<\/span><span class=\"p\">:<\/span>\n        <span class=\"n\">internal_link<\/span> <span class=\"o\">=<\/span> <span class=\"n\">bs<\/span><span class=\"p\">.<\/span><span class=\"n\">find_all<\/span><span class=\"p\">(<\/span><span class=\"s\">\"a\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">hostname<\/span> <span class=\"o\">=<\/span> <span class=\"n\">urlsplit<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">).<\/span><span class=\"n\">hostname<\/span>\n        <span class=\"k\">for<\/span> <span class=\"n\">link<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">internal_link<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">valid_link<\/span> <span class=\"o\">=<\/span> <span class=\"n\">link<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"href\"<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">if<\/span> <span class=\"n\">valid_link<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"http:\/\/\"<\/span><span class=\"p\">)<\/span> <span class=\"ow\">or<\/span> <span class=\"n\">valid_link<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"https:\/\/\"<\/span><span class=\"p\">):<\/span>\n                <span class=\"k\">if<\/span> <span class=\"n\">urlsplit<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_link<\/span><span class=\"p\">).<\/span><span class=\"n\">hostname<\/span> <span class=\"o\">==<\/span> <span class=\"n\">hostname<\/span><span class=\"p\">:<\/span>\n                    <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">internal_links<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">valid_link<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">internal_links<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">scrape_image<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">bs<\/span><span class=\"p\">:<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"nb\">list<\/span><span class=\"p\">:<\/span>\n        <span class=\"s\">\"\"\"scrape images data\n\n        Args:\n            bs (BeautifulSoup): beautifulsoup object\n\n        Returns:\n            list: return list of images\n        \"\"\"<\/span>\n        <span class=\"n\">images<\/span> <span class=\"o\">=<\/span> <span class=\"n\">bs<\/span><span class=\"p\">.<\/span><span class=\"n\">find_all<\/span><span class=\"p\">(<\/span><span class=\"s\">\"img\"<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">for<\/span> <span class=\"n\">img<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">images<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">source<\/span> <span class=\"o\">=<\/span> <span class=\"n\">img<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"alt\"<\/span><span class=\"p\">)<\/span>\n\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">images<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">source<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">images<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">talk<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">]:<\/span>\n        <span class=\"s\">\"\"\"Returns a dictionary that can be printed\n\n        Returns:\n            dict: dictionary context data that can be printed\n        \"\"\"<\/span>\n        <span class=\"n\">context<\/span><span class=\"p\">:<\/span> <span class=\"nb\">dict<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{<\/span>\n            <span class=\"s\">\"url\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"title\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">title<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"description\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">description<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"word_count\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">total_word_count<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"keywords\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">sort_freq_dist<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">keywords<\/span><span class=\"p\">,<\/span> <span class=\"n\">limit<\/span><span class=\"o\">=<\/span><span class=\"mi\">5<\/span><span class=\"p\">),<\/span>\n            <span class=\"s\">\"bigrams\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">bigrams<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"trigrams\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">trigrams<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"broken links\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">broken_link<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"internal links\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">internal_links<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"external links\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">external_link<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"warnings\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">warnings<\/span><span class=\"p\">,<\/span>\n            <span class=\"s\">\"content_hash\"<\/span><span class=\"p\">:<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hash<\/span><span class=\"p\">,<\/span>\n        <span class=\"p\">}<\/span>\n\n        <span class=\"c1\"># append new items\n<\/span>        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">context<\/span><span class=\"p\">[<\/span><span class=\"s\">\"headings\"<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">headings<\/span>\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">context<\/span><span class=\"p\">[<\/span><span class=\"s\">\"additional_info\"<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">additional_info<\/span>\n\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">context<\/span><span class=\"p\">[<\/span><span class=\"s\">\"images\"<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">images<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"n\">context<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">analyze<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">raw_html<\/span><span class=\"o\">=<\/span><span class=\"bp\">None<\/span><span class=\"p\">):<\/span>\n        <span class=\"s\">\"\"\"\n        Analyze the page and populate the warnings list\n        \"\"\"<\/span>\n\n        <span class=\"k\">if<\/span> <span class=\"ow\">not<\/span> <span class=\"n\">raw_html<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">valid_prefixes<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n\n            <span class=\"c1\"># only allow http:\/\/ https:\/\/ and \/\/\n<\/span>            <span class=\"k\">for<\/span> <span class=\"n\">s<\/span> <span class=\"ow\">in<\/span> <span class=\"p\">[<\/span>\n                <span class=\"s\">\"http:\/\/\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"https:\/\/\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"\/\/\"<\/span><span class=\"p\">,<\/span>\n            <span class=\"p\">]:<\/span>\n                <span class=\"n\">valid_prefixes<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"n\">s<\/span><span class=\"p\">))<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"bp\">True<\/span> <span class=\"ow\">not<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">valid_prefixes<\/span><span class=\"p\">:<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">warn<\/span><span class=\"p\">(<\/span><span class=\"sa\">f<\/span><span class=\"s\">\"<\/span><span class=\"si\">{<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"si\">}<\/span><span class=\"s\"> does not appear to have a valid protocol.\"<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">return<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">.<\/span><span class=\"n\">startswith<\/span><span class=\"p\">(<\/span><span class=\"s\">\"\/\/\"<\/span><span class=\"p\">):<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span> <span class=\"o\">=<\/span> <span class=\"sa\">f<\/span><span class=\"s\">\"<\/span><span class=\"si\">{<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">base_domain<\/span><span class=\"p\">.<\/span><span class=\"n\">scheme<\/span><span class=\"si\">}<\/span><span class=\"s\">:<\/span><span class=\"si\">{<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"si\">}<\/span><span class=\"s\">\"<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">parsed_url<\/span><span class=\"p\">.<\/span><span class=\"n\">netloc<\/span> <span class=\"o\">!=<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">base_domain<\/span><span class=\"p\">.<\/span><span class=\"n\">netloc<\/span><span class=\"p\">:<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">warn<\/span><span class=\"p\">(<\/span><span class=\"sa\">f<\/span><span class=\"s\">\"<\/span><span class=\"si\">{<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"si\">}<\/span><span class=\"s\"> is not part of <\/span><span class=\"si\">{<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">base_domain<\/span><span class=\"p\">.<\/span><span class=\"n\">netloc<\/span><span class=\"si\">}<\/span><span class=\"s\">.\"<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">return<\/span>\n\n            <span class=\"k\">try<\/span><span class=\"p\">:<\/span>\n                <span class=\"n\">page<\/span> <span class=\"o\">=<\/span> <span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">except<\/span> <span class=\"n\">HTTPError<\/span> <span class=\"k\">as<\/span> <span class=\"n\">e<\/span><span class=\"p\">:<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">warn<\/span><span class=\"p\">(<\/span><span class=\"sa\">f<\/span><span class=\"s\">\"Returned <\/span><span class=\"si\">{<\/span><span class=\"n\">e<\/span><span class=\"si\">}<\/span><span class=\"s\">\"<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">return<\/span>\n\n            <span class=\"n\">encoding<\/span> <span class=\"o\">=<\/span> <span class=\"s\">\"ascii\"<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"s\">\"content-type\"<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">headers<\/span><span class=\"p\">:<\/span>\n                <span class=\"n\">encoding<\/span> <span class=\"o\">=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">headers<\/span><span class=\"p\">[<\/span><span class=\"s\">\"content-type\"<\/span><span class=\"p\">].<\/span><span class=\"n\">split<\/span><span class=\"p\">(<\/span><span class=\"s\">\"charset=\"<\/span><span class=\"p\">)[<\/span><span class=\"o\">-<\/span><span class=\"mi\">1<\/span><span class=\"p\">]<\/span>\n\n            <span class=\"k\">if<\/span> <span class=\"n\">encoding<\/span><span class=\"p\">.<\/span><span class=\"n\">lower<\/span><span class=\"p\">()<\/span> <span class=\"ow\">not<\/span> <span class=\"ow\">in<\/span> <span class=\"p\">(<\/span><span class=\"s\">\"text\/html\"<\/span><span class=\"p\">,<\/span> <span class=\"s\">\"text\/plain\"<\/span><span class=\"p\">,<\/span> <span class=\"s\">\"utf-8\"<\/span><span class=\"p\">):<\/span>\n                <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">warn<\/span><span class=\"p\">(<\/span><span class=\"sa\">f<\/span><span class=\"s\">\"Can not read <\/span><span class=\"si\">{<\/span><span class=\"n\">encoding<\/span><span class=\"si\">}<\/span><span class=\"s\">\"<\/span><span class=\"p\">)<\/span>\n                <span class=\"k\">return<\/span>\n            <span class=\"k\">else<\/span><span class=\"p\">:<\/span>\n                <span class=\"n\">raw_html<\/span> <span class=\"o\">=<\/span> <span class=\"n\">page<\/span><span class=\"p\">.<\/span><span class=\"n\">data<\/span><span class=\"p\">.<\/span><span class=\"n\">decode<\/span><span class=\"p\">(<\/span><span class=\"s\">\"utf-8\"<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hash<\/span> <span class=\"o\">=<\/span> <span class=\"n\">hashlib<\/span><span class=\"p\">.<\/span><span class=\"n\">sha1<\/span><span class=\"p\">(<\/span><span class=\"n\">raw_html<\/span><span class=\"p\">.<\/span><span class=\"n\">encode<\/span><span class=\"p\">(<\/span><span class=\"s\">\"utf-8\"<\/span><span class=\"p\">)).<\/span><span class=\"n\">hexdigest<\/span><span class=\"p\">()<\/span>\n\n        <span class=\"c1\"># remove comments, they screw with BeautifulSoup\n<\/span>        <span class=\"n\">clean_html<\/span> <span class=\"o\">=<\/span> <span class=\"n\">re<\/span><span class=\"p\">.<\/span><span class=\"n\">sub<\/span><span class=\"p\">(<\/span><span class=\"sa\">r<\/span><span class=\"s\">\"&lt;!--.*?--&gt;\"<\/span><span class=\"p\">,<\/span> <span class=\"sa\">r<\/span><span class=\"s\">\"\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">raw_html<\/span><span class=\"p\">,<\/span> <span class=\"n\">flags<\/span><span class=\"o\">=<\/span><span class=\"n\">re<\/span><span class=\"p\">.<\/span><span class=\"n\">DOTALL<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"n\">soup_lower<\/span> <span class=\"o\">=<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">(<\/span>\n            <span class=\"n\">clean_html<\/span><span class=\"p\">.<\/span><span class=\"n\">lower<\/span><span class=\"p\">(),<\/span> <span class=\"s\">\"html.parser\"<\/span>\n        <span class=\"p\">)<\/span>  <span class=\"c1\"># .encode('utf-8')\n<\/span>        <span class=\"n\">soup_unmodified<\/span> <span class=\"o\">=<\/span> <span class=\"n\">BeautifulSoup<\/span><span class=\"p\">(<\/span><span class=\"n\">clean_html<\/span><span class=\"p\">,<\/span> <span class=\"s\">\"html.parser\"<\/span><span class=\"p\">)<\/span>  <span class=\"c1\"># .encode('utf-8')\n<\/span>\n        <span class=\"n\">texts<\/span> <span class=\"o\">=<\/span> <span class=\"n\">soup_lower<\/span><span class=\"p\">.<\/span><span class=\"n\">findAll<\/span><span class=\"p\">(<\/span><span class=\"n\">text<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span>\n        <span class=\"n\">visible_text<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[<\/span><span class=\"n\">w<\/span> <span class=\"k\">for<\/span> <span class=\"n\">w<\/span> <span class=\"ow\">in<\/span> <span class=\"nb\">filter<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">visible_tags<\/span><span class=\"p\">,<\/span> <span class=\"n\">texts<\/span><span class=\"p\">)]<\/span>\n\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">process_text<\/span><span class=\"p\">(<\/span><span class=\"n\">visible_text<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">populate<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_lower<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_title<\/span><span class=\"p\">()<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_description<\/span><span class=\"p\">()<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_og<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_lower<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_a_tags<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_img_tags<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_lower<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_h1_tags<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_lower<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"c1\"># add broken and alt image text wrapper\n<\/span>        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_broken_link<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_internal_link<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_external_link<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_heading_tags<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">analyze_additional_tags<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"c1\"># add scrape image function\n<\/span>        <span class=\"k\">if<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">:<\/span>\n            <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">scrape_image<\/span><span class=\"p\">(<\/span><span class=\"n\">soup_unmodified<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"k\">return<\/span> <span class=\"bp\">True<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h3><span class=\"ez-toc-section\" id=\"%D9%86%D9%85%D9%88%D9%86%D9%87_%D8%A2%D9%86%D8%A7%D9%84%D8%A7%DB%8C%D8%B2%D8%B1_%D8%B3%D8%A7%D8%AE%D8%AA\"><\/span>\n<p>  \u0646\u0645\u0648\u0646\u0647 \u0622\u0646\u0627\u0644\u0627\u06cc\u0632\u0631 \u0633\u0627\u062e\u062a<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0628\u0631\u0627\u06cc \u0627\u062c\u0631\u0627\u06cc \u0645\u0627\u0698\u0648\u0644 \u0627\u0633\u06a9\u0631\u0627\u067e\u0631\u060c \u0628\u06cc\u0627\u06cc\u06cc\u062f \u0646\u0645\u0648\u0646\u0647 \u0622\u0646\u0627\u0644\u06cc\u0632\u0648\u0631 \u0631\u0627 \u0628\u0631\u0627\u06cc \u062a\u0631\u06a9\u06cc\u0628 \u06cc\u06a9 \u0645\u0627\u0698\u0648\u0644 \u0633\u0641\u0627\u0631\u0634\u06cc \u0628\u0633\u0627\u0632\u06cc\u0645<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/modules\/analyzer.py\n<\/span><span class=\"kn\">import<\/span> <span class=\"nn\">time<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Any<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">operator<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">itemgetter<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">modules.crawler<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">WebCrawler<\/span>\n\n\n<span class=\"k\">def<\/span> <span class=\"nf\">analyze<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">sitemap_url<\/span><span class=\"o\">=<\/span><span class=\"bp\">None<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_headings<\/span><span class=\"o\">=<\/span><span class=\"bp\">False<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"o\">=<\/span><span class=\"bp\">False<\/span><span class=\"p\">,<\/span> <span class=\"n\">follow_links<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">,<\/span> <span class=\"n\">scrape_img<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">]:<\/span>\n    <span class=\"n\">start_time<\/span> <span class=\"o\">=<\/span> <span class=\"n\">time<\/span><span class=\"p\">.<\/span><span class=\"n\">time<\/span><span class=\"p\">()<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">calc_total_time<\/span><span class=\"p\">():<\/span>\n        <span class=\"k\">return<\/span> <span class=\"n\">time<\/span><span class=\"p\">.<\/span><span class=\"n\">time<\/span><span class=\"p\">()<\/span> <span class=\"o\">-<\/span> <span class=\"n\">start_time<\/span>\n\n    <span class=\"n\">output<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{<\/span><span class=\"s\">'pages'<\/span><span class=\"p\">:<\/span> <span class=\"p\">[],<\/span> <span class=\"s\">'keywords'<\/span><span class=\"p\">:<\/span> <span class=\"p\">[],<\/span> <span class=\"s\">'errors'<\/span><span class=\"p\">:<\/span> <span class=\"p\">[],<\/span> <span class=\"s\">'total_time'<\/span><span class=\"p\">:<\/span> <span class=\"n\">calc_total_time<\/span><span class=\"p\">()}<\/span>\n\n    <span class=\"n\">site<\/span> <span class=\"o\">=<\/span> <span class=\"n\">WebCrawler<\/span><span class=\"p\">(<\/span><span class=\"n\">base_url<\/span><span class=\"o\">=<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">sitemap<\/span><span class=\"o\">=<\/span><span class=\"n\">sitemap_url<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_headings<\/span><span class=\"o\">=<\/span><span class=\"n\">analyze_headings<\/span><span class=\"p\">,<\/span> <span class=\"n\">analyze_extra_tags<\/span><span class=\"o\">=<\/span><span class=\"n\">analyze_extra_tags<\/span><span class=\"p\">,<\/span> <span class=\"n\">follow_links<\/span><span class=\"o\">=<\/span><span class=\"n\">follow_links<\/span><span class=\"p\">,<\/span> <span class=\"n\">scrape_img<\/span><span class=\"o\">=<\/span><span class=\"n\">scrape_img<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">crawl<\/span><span class=\"p\">()<\/span>\n\n    <span class=\"k\">for<\/span> <span class=\"n\">p<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">crawled_pages<\/span><span class=\"p\">:<\/span>\n        <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'pages'<\/span><span class=\"p\">].<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">p<\/span><span class=\"p\">.<\/span><span class=\"n\">talk<\/span><span class=\"p\">())<\/span>\n\n    <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'duplicate_pages'<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[<\/span><span class=\"nb\">list<\/span><span class=\"p\">(<\/span><span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hashes<\/span><span class=\"p\">[<\/span><span class=\"n\">p<\/span><span class=\"p\">])<\/span> <span class=\"k\">for<\/span> <span class=\"n\">p<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hashes<\/span> <span class=\"k\">if<\/span> <span class=\"nb\">len<\/span><span class=\"p\">(<\/span><span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">content_hashes<\/span><span class=\"p\">[<\/span><span class=\"n\">p<\/span><span class=\"p\">])<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">1<\/span><span class=\"p\">]<\/span>\n\n    <span class=\"n\">sorted_words<\/span> <span class=\"o\">=<\/span> <span class=\"nb\">sorted<\/span><span class=\"p\">(<\/span><span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">wordcount<\/span><span class=\"p\">.<\/span><span class=\"n\">items<\/span><span class=\"p\">(),<\/span> <span class=\"n\">key<\/span><span class=\"o\">=<\/span><span class=\"n\">itemgetter<\/span><span class=\"p\">(<\/span><span class=\"mi\">1<\/span><span class=\"p\">),<\/span> <span class=\"n\">reverse<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span>\n    <span class=\"n\">sorted_bigrams<\/span> <span class=\"o\">=<\/span> <span class=\"nb\">sorted<\/span><span class=\"p\">(<\/span><span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">bigrams<\/span><span class=\"p\">.<\/span><span class=\"n\">items<\/span><span class=\"p\">(),<\/span> <span class=\"n\">key<\/span><span class=\"o\">=<\/span><span class=\"n\">itemgetter<\/span><span class=\"p\">(<\/span><span class=\"mi\">1<\/span><span class=\"p\">),<\/span> <span class=\"n\">reverse<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span>\n    <span class=\"n\">sorted_trigrams<\/span> <span class=\"o\">=<\/span> <span class=\"nb\">sorted<\/span><span class=\"p\">(<\/span><span class=\"n\">site<\/span><span class=\"p\">.<\/span><span class=\"n\">trigrams<\/span><span class=\"p\">.<\/span><span class=\"n\">items<\/span><span class=\"p\">(),<\/span> <span class=\"n\">key<\/span><span class=\"o\">=<\/span><span class=\"n\">itemgetter<\/span><span class=\"p\">(<\/span><span class=\"mi\">1<\/span><span class=\"p\">),<\/span> <span class=\"n\">reverse<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[]<\/span>\n\n    <span class=\"k\">for<\/span> <span class=\"n\">w<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">sorted_words<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">if<\/span> <span class=\"n\">w<\/span><span class=\"p\">[<\/span><span class=\"mi\">1<\/span><span class=\"p\">]<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">4<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">].<\/span><span class=\"n\">append<\/span><span class=\"p\">({<\/span>\n                <span class=\"s\">'word'<\/span><span class=\"p\">:<\/span> <span class=\"n\">w<\/span><span class=\"p\">[<\/span><span class=\"mi\">0<\/span><span class=\"p\">],<\/span>\n                <span class=\"s\">'count'<\/span><span class=\"p\">:<\/span> <span class=\"n\">w<\/span><span class=\"p\">[<\/span><span class=\"mi\">1<\/span><span class=\"p\">],<\/span>\n            <span class=\"p\">})<\/span>\n\n    <span class=\"k\">for<\/span> <span class=\"n\">w<\/span><span class=\"p\">,<\/span> <span class=\"n\">v<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">sorted_bigrams<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">if<\/span> <span class=\"n\">v<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">4<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">].<\/span><span class=\"n\">append<\/span><span class=\"p\">({<\/span>\n                <span class=\"s\">'word'<\/span><span class=\"p\">:<\/span> <span class=\"n\">w<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">'count'<\/span><span class=\"p\">:<\/span> <span class=\"n\">v<\/span><span class=\"p\">,<\/span>\n            <span class=\"p\">})<\/span>\n\n    <span class=\"k\">for<\/span> <span class=\"n\">w<\/span><span class=\"p\">,<\/span> <span class=\"n\">v<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">sorted_trigrams<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">if<\/span> <span class=\"n\">v<\/span> <span class=\"o\">&gt;<\/span> <span class=\"mi\">4<\/span><span class=\"p\">:<\/span>\n            <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">].<\/span><span class=\"n\">append<\/span><span class=\"p\">({<\/span>\n                <span class=\"s\">'word'<\/span><span class=\"p\">:<\/span> <span class=\"n\">w<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">'count'<\/span><span class=\"p\">:<\/span> <span class=\"n\">v<\/span><span class=\"p\">,<\/span>\n            <span class=\"p\">})<\/span>\n\n    <span class=\"c1\"># Sort one last time...\n<\/span>    <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"nb\">sorted<\/span><span class=\"p\">(<\/span><span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'keywords'<\/span><span class=\"p\">],<\/span> <span class=\"n\">key<\/span><span class=\"o\">=<\/span><span class=\"n\">itemgetter<\/span><span class=\"p\">(<\/span><span class=\"s\">'count'<\/span><span class=\"p\">),<\/span> <span class=\"n\">reverse<\/span><span class=\"o\">=<\/span><span class=\"bp\">True<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"n\">output<\/span><span class=\"p\">[<\/span><span class=\"s\">'total_time'<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"n\">calc_total_time<\/span><span class=\"p\">()<\/span>\n\n    <span class=\"k\">return<\/span> <span class=\"n\">output<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h3><span class=\"ez-toc-section\" id=\"%D8%A7%D8%B6%D8%A7%D9%81%D9%87_%DA%A9%D8%B1%D8%AF%D9%86_%D9%85%D8%A7%DA%98%D9%88%D9%84_%D9%87%D8%A7%DB%8C_%DA%A9%D9%85%DA%A9%DB%8C\"><\/span>\n<p>  \u0627\u0636\u0627\u0641\u0647 \u06a9\u0631\u062f\u0646 \u0645\u0627\u0698\u0648\u0644 \u0647\u0627\u06cc \u06a9\u0645\u06a9\u06cc<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u0627\u06cc\u0646 \u0631\u0627 \u0627\u0636\u0627\u0641\u0647 \u06a9\u0646\u06cc\u062f <code>helper.py<\/code> \u0628\u0631\u0627\u06cc \u0627\u06cc\u062c\u0627\u062f \u0627\u0628\u0632\u0627\u0631 \u0628\u0631\u0627\u06cc \u0645\u0627\u0698\u0648\u0644 \u0627\u0633\u06a9\u0631\u0627\u067e\u0631<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/modules\/helper.py\n<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">certifi<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">requests<\/span>\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib3<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">PoolManager<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib3<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Timeout<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib3.response<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">HTTPResponse<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">urllib3.util<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Retry<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Any<\/span>\n\n\n\n<span class=\"k\">class<\/span> <span class=\"nc\">Http<\/span><span class=\"p\">:<\/span>\n    <span class=\"k\">def<\/span> <span class=\"nf\">__init__<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">):<\/span>\n        <span class=\"n\">user_agent<\/span><span class=\"p\">:<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"nb\">str<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{<\/span>\n            <span class=\"s\">\"User-Agent\"<\/span><span class=\"p\">:<\/span> <span class=\"s\">\"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/108.0.0.0 Safari\/537.36\"<\/span>\n        <span class=\"p\">}<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">retry_strategy<\/span><span class=\"p\">:<\/span> <span class=\"n\">Retry<\/span> <span class=\"o\">=<\/span> <span class=\"n\">Retry<\/span><span class=\"p\">(<\/span>\n            <span class=\"n\">total<\/span><span class=\"o\">=<\/span><span class=\"mi\">10<\/span><span class=\"p\">,<\/span>  <span class=\"c1\"># maximum number of retries\n<\/span>            <span class=\"n\">redirect<\/span><span class=\"o\">=<\/span><span class=\"mi\">100<\/span><span class=\"p\">,<\/span>  <span class=\"c1\"># maximum number of redirects to follow\n<\/span>            <span class=\"n\">status_forcelist<\/span><span class=\"o\">=<\/span><span class=\"p\">[<\/span>\n                <span class=\"mi\">429<\/span><span class=\"p\">,<\/span>\n                <span class=\"mi\">500<\/span><span class=\"p\">,<\/span>\n                <span class=\"mi\">502<\/span><span class=\"p\">,<\/span>\n                <span class=\"mi\">503<\/span><span class=\"p\">,<\/span>\n                <span class=\"mi\">504<\/span><span class=\"p\">,<\/span>\n            <span class=\"p\">],<\/span>  <span class=\"c1\"># list of HTTP status codes to retry on\n<\/span>            <span class=\"n\">method_whitelist<\/span><span class=\"o\">=<\/span><span class=\"p\">[<\/span>\n                <span class=\"s\">\"HEAD\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"TRACE\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"GET\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"PUT\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"OPTIONS\"<\/span><span class=\"p\">,<\/span>\n                <span class=\"s\">\"DELETE\"<\/span><span class=\"p\">,<\/span>\n            <span class=\"p\">],<\/span>  <span class=\"c1\"># list of HTTP methods to retry on\n<\/span>        <span class=\"p\">)<\/span>\n        <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">http<\/span><span class=\"p\">:<\/span> <span class=\"n\">PoolManager<\/span> <span class=\"o\">=<\/span> <span class=\"n\">PoolManager<\/span><span class=\"p\">(<\/span>\n            <span class=\"n\">timeout<\/span><span class=\"o\">=<\/span><span class=\"n\">Timeout<\/span><span class=\"p\">(<\/span><span class=\"n\">connect<\/span><span class=\"o\">=<\/span><span class=\"mf\">1.0<\/span><span class=\"p\">,<\/span> <span class=\"n\">read<\/span><span class=\"o\">=<\/span><span class=\"mf\">2.0<\/span><span class=\"p\">),<\/span>\n            <span class=\"n\">cert_reqs<\/span><span class=\"o\">=<\/span><span class=\"s\">\"CERT_REQUIRED\"<\/span><span class=\"p\">,<\/span>\n            <span class=\"n\">ca_certs<\/span><span class=\"o\">=<\/span><span class=\"n\">certifi<\/span><span class=\"p\">.<\/span><span class=\"n\">where<\/span><span class=\"p\">(),<\/span>\n            <span class=\"n\">headers<\/span><span class=\"o\">=<\/span><span class=\"n\">user_agent<\/span><span class=\"p\">,<\/span>\n            <span class=\"n\">retries<\/span><span class=\"o\">=<\/span><span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">retry_strategy<\/span><span class=\"p\">,<\/span>\n        <span class=\"p\">)<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">get<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">Any<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">request<\/span><span class=\"p\">(<\/span><span class=\"s\">\"GET\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">get_without_redirect<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">HTTPResponse<\/span><span class=\"p\">:<\/span>\n        <span class=\"n\">res<\/span> <span class=\"o\">=<\/span> <span class=\"n\">requests<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">verify<\/span><span class=\"o\">=<\/span><span class=\"bp\">False<\/span><span class=\"p\">)<\/span>\n        <span class=\"k\">return<\/span> <span class=\"n\">res<\/span>\n\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">post<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">data<\/span><span class=\"p\">:<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">])<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">Any<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">request<\/span><span class=\"p\">(<\/span><span class=\"s\">\"POST\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">fields<\/span><span class=\"o\">=<\/span><span class=\"n\">data<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">put<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">data<\/span><span class=\"p\">:<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">])<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">HTTPResponse<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">request<\/span><span class=\"p\">(<\/span><span class=\"s\">\"PUT\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">,<\/span> <span class=\"n\">fields<\/span><span class=\"o\">=<\/span><span class=\"n\">data<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"k\">def<\/span> <span class=\"nf\">delete<\/span><span class=\"p\">(<\/span><span class=\"bp\">self<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">:<\/span> <span class=\"nb\">str<\/span><span class=\"p\">)<\/span> <span class=\"o\">-&gt;<\/span> <span class=\"n\">HTTPResponse<\/span><span class=\"p\">:<\/span>\n        <span class=\"k\">return<\/span> <span class=\"bp\">self<\/span><span class=\"p\">.<\/span><span class=\"n\">http<\/span><span class=\"p\">.<\/span><span class=\"n\">request<\/span><span class=\"p\">(<\/span><span class=\"s\">\"DELETE\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">url<\/span><span class=\"p\">)<\/span>\n\n\n<span class=\"n\">http<\/span> <span class=\"o\">=<\/span> <span class=\"n\">Http<\/span><span class=\"p\">()<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<p>\u0648 \u062e\u0648\u0628 \u0627\u0646\u062c\u0627\u0645 \u0634\u062f\u060c \u0645\u0627\u0698\u0648\u0644 \u0627\u0633\u06a9\u0631\u0627\u067e\u0631 \u0622\u0645\u0627\u062f\u0647 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0633\u062a\u060c \u0628\u0627 \u0639\u0645\u0644\u06a9\u0631\u062f \u0633\u0641\u0627\u0631\u0634\u06cc \u0642\u0627\u0628\u0644 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0645\u062c\u062f\u062f\u060c \u0627\u062c\u0627\u0632\u0647 \u062f\u0647\u06cc\u062f \u0628\u0647 \u0645\u0631\u062d\u0644\u0647 \u0628\u0639\u062f\u06cc \u0627\u062f\u0627\u0645\u0647 \u062f\u0647\u06cc\u0645\u060c \u0628\u06cc\u0627\u06cc\u06cc\u062f \u0628\u0631\u0646\u0627\u0645\u0647 \u0648\u0628 \u0631\u0627 \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 \u0641\u0644\u0627\u0633\u06a9 \u0628\u0633\u0627\u0632\u06cc\u0645<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%D8%B3%D8%A7%D8%AE%D8%AA_%D8%A8%D8%B1%D9%86%D8%A7%D9%85%D9%87_%D9%87%D8%A7%DB%8C_%D9%88%D8%A8_%D8%A8%D8%A7_%D8%A7%D8%B3%D8%AA%D9%81%D8%A7%D8%AF%D9%87_%D8%A7%D8%B2_Flask\"><\/span>\n<p>  \u0633\u0627\u062e\u062a \u0628\u0631\u0646\u0627\u0645\u0647 \u0647\u0627\u06cc \u0648\u0628 \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 Flask<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u062f\u0631 \u0645\u0631\u062d\u0644\u0647 \u0642\u0628\u0644 \u0628\u0631\u0646\u0627\u0645\u0647 \u0641\u0644\u0627\u0633\u06a9 \u0631\u0627 \u0631\u0627\u0647 \u0627\u0646\u062f\u0627\u0632\u06cc \u06a9\u0631\u062f\u06cc\u0645\u060c \u0628\u0646\u0627\u0628\u0631\u0627\u06cc\u0646 \u062f\u0631 \u0627\u06cc\u0646 \u0645\u0631\u062d\u0644\u0647 \u0645\u0633\u06cc\u0631\u0647\u0627 \u0631\u0627 \u0645\u06cc \u0633\u0627\u0632\u06cc\u0645 \u0648 \u0628\u0631\u0646\u0627\u0645\u0647 \u0647\u0627\u06cc \u0648\u0628 \u0631\u0627 \u0645\u0634\u0627\u0647\u062f\u0647 \u0645\u06cc \u06a9\u0646\u06cc\u0645.<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight python\"><code><span class=\"c1\"># src\/apps\/home\/views.py\n<\/span>\n<span class=\"kn\">import<\/span> <span class=\"nn\">json<\/span>\n\n<span class=\"c1\"># from seoanalyzer import analyze\n<\/span><span class=\"kn\">from<\/span> <span class=\"nn\">flask<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Blueprint<\/span><span class=\"p\">,<\/span> <span class=\"n\">render_template<\/span><span class=\"p\">,<\/span> <span class=\"n\">request<\/span><span class=\"p\">,<\/span> <span class=\"n\">jsonify<\/span>\n<span class=\"kn\">from<\/span> <span class=\"nn\">typing<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">Any<\/span>\n\n\n<span class=\"kn\">from<\/span> <span class=\"nn\">modules.analyzer<\/span> <span class=\"kn\">import<\/span> <span class=\"n\">analyze<\/span>\n\n<span class=\"n\">home_bp<\/span><span class=\"p\">:<\/span> <span class=\"n\">Blueprint<\/span> <span class=\"o\">=<\/span> <span class=\"n\">Blueprint<\/span><span class=\"p\">(<\/span>\n    <span class=\"s\">\"home\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">__name__<\/span><span class=\"p\">,<\/span> <span class=\"n\">template_folder<\/span><span class=\"o\">=<\/span><span class=\"s\">\"templates\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">static_folder<\/span><span class=\"o\">=<\/span><span class=\"s\">\"static\"<\/span>\n<span class=\"p\">)<\/span>\n\n\n<span class=\"o\">@<\/span><span class=\"n\">home_bp<\/span><span class=\"p\">.<\/span><span class=\"n\">route<\/span><span class=\"p\">(<\/span><span class=\"s\">\"https:\/\/dev.to\/\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">methods<\/span><span class=\"o\">=<\/span><span class=\"p\">[<\/span><span class=\"s\">\"GET\"<\/span><span class=\"p\">,<\/span> <span class=\"s\">\"POST\"<\/span><span class=\"p\">])<\/span>\n<span class=\"k\">def<\/span> <span class=\"nf\">index<\/span><span class=\"p\">():<\/span>\n    <span class=\"k\">if<\/span> <span class=\"n\">request<\/span><span class=\"p\">.<\/span><span class=\"n\">method<\/span> <span class=\"o\">==<\/span> <span class=\"s\">\"POST\"<\/span><span class=\"p\">:<\/span>\n        <span class=\"n\">query<\/span> <span class=\"o\">=<\/span> <span class=\"n\">request<\/span><span class=\"p\">.<\/span><span class=\"n\">form<\/span><span class=\"p\">.<\/span><span class=\"n\">get<\/span><span class=\"p\">(<\/span><span class=\"s\">\"website\"<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"n\">analyzer<\/span> <span class=\"o\">=<\/span> <span class=\"n\">analyze<\/span><span class=\"p\">(<\/span>\n            <span class=\"n\">url<\/span><span class=\"o\">=<\/span><span class=\"n\">query<\/span><span class=\"p\">,<\/span>\n            <span class=\"n\">follow_links<\/span><span class=\"o\">=<\/span><span class=\"bp\">False<\/span><span class=\"p\">,<\/span>\n        <span class=\"p\">)<\/span>\n        <span class=\"n\">results<\/span><span class=\"p\">:<\/span> <span class=\"nb\">list<\/span> <span class=\"o\">=<\/span> <span class=\"p\">[<\/span><span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">]]<\/span>\n\n        <span class=\"c1\"># process pages here\n<\/span>        <span class=\"k\">for<\/span> <span class=\"n\">page<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">analyzer<\/span><span class=\"p\">[<\/span><span class=\"s\">\"pages\"<\/span><span class=\"p\">]:<\/span>\n            <span class=\"n\">data_dict<\/span><span class=\"p\">:<\/span> <span class=\"nb\">dict<\/span><span class=\"p\">[<\/span><span class=\"nb\">str<\/span><span class=\"p\">,<\/span> <span class=\"n\">Any<\/span><span class=\"p\">]<\/span> <span class=\"o\">=<\/span> <span class=\"p\">{<\/span>\n                <span class=\"s\">\"word count\"<\/span><span class=\"p\">:<\/span> <span class=\"n\">page<\/span><span class=\"p\">[<\/span><span class=\"s\">\"word_count\"<\/span><span class=\"p\">],<\/span>\n                <span class=\"s\">\"page title\"<\/span><span class=\"p\">:<\/span> <span class=\"n\">page<\/span><span class=\"p\">[<\/span><span class=\"s\">\"title\"<\/span><span class=\"p\">],<\/span>\n            <span class=\"p\">}<\/span>\n            <span class=\"n\">results<\/span><span class=\"p\">.<\/span><span class=\"n\">append<\/span><span class=\"p\">(<\/span><span class=\"n\">data_dict<\/span><span class=\"p\">)<\/span>\n            <span class=\"k\">for<\/span> <span class=\"n\">keywords<\/span> <span class=\"ow\">in<\/span> <span class=\"n\">page<\/span><span class=\"p\">[<\/span><span class=\"s\">\"keywords\"<\/span><span class=\"p\">]:<\/span>\n                <span class=\"k\">print<\/span><span class=\"p\">(<\/span><span class=\"n\">keywords<\/span><span class=\"p\">)<\/span>\n\n        <span class=\"c1\"># (['pages', 'keywords', 'errors', 'total_time', 'duplicate_pages']\n<\/span>\n        <span class=\"c1\"># return json\n<\/span>        <span class=\"k\">return<\/span> <span class=\"n\">render_template<\/span><span class=\"p\">(<\/span><span class=\"s\">\"index.html\"<\/span><span class=\"p\">,<\/span> <span class=\"n\">datas<\/span><span class=\"o\">=<\/span><span class=\"n\">results<\/span><span class=\"p\">)<\/span>\n\n    <span class=\"k\">return<\/span> <span class=\"n\">render_template<\/span><span class=\"p\">(<\/span><span class=\"s\">\"index.html\"<\/span><span class=\"p\">)<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<h3><span class=\"ez-toc-section\" id=\"%D9%82%D8%A7%D9%84%D8%A8_%D9%87%D8%A7_%D8%B1%D8%A7_%D8%B7%D8%B1%D8%A7%D8%AD%DB%8C_%DA%A9%D9%86%DB%8C%D8%AF\"><\/span>\n<p>  \u0642\u0627\u0644\u0628 \u0647\u0627 \u0631\u0627 \u0637\u0631\u0627\u062d\u06cc \u06a9\u0646\u06cc\u062f<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u067e\u0633 \u0627\u0632 \u0637\u0631\u0627\u062d\u06cc \u0646\u0645\u0627\u0647\u0627\u060c \u0628\u06cc\u0627\u06cc\u06cc\u062f \u0628\u0627 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 bootstrap in \u06cc\u06a9 \u0642\u0627\u0644\u0628 \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u06cc\u0645 <code>index.html<\/code> \u06a9\u0647 \u062f\u0631 \u0627\u06cc\u062c\u0627\u062f \u0634\u062f\u0647 \u0627\u0633\u062a <code>src\/apps\/home\/templates<\/code> \u0641\u0647\u0631\u0633\u062a \u0631\u0627\u0647\u0646\u0645\u0627<\/p>\n<div class=\"highlight js-code-highlight\">\n<pre class=\"highlight html\"><code><span class=\"cp\">&lt;!doctype html&gt;<\/span>\n<span class=\"nt\">&lt;html<\/span> <span class=\"na\">lang=<\/span><span class=\"s\">\"en\"<\/span><span class=\"nt\">&gt;<\/span>\n  <span class=\"nt\">&lt;head&gt;<\/span>\n    <span class=\"nt\">&lt;meta<\/span> <span class=\"na\">charset=<\/span><span class=\"s\">\"utf-8\"<\/span><span class=\"nt\">&gt;<\/span>\n    <span class=\"nt\">&lt;meta<\/span> <span class=\"na\">name=<\/span><span class=\"s\">\"viewport\"<\/span> <span class=\"na\">content=<\/span><span class=\"s\">\"width=device-width, initial-scale=1\"<\/span><span class=\"nt\">&gt;<\/span>\n    <span class=\"nt\">&lt;title&gt;<\/span>SEO Audit Tools<span class=\"nt\">&lt;\/title&gt;<\/span>\n    <span class=\"nt\">&lt;link<\/span> <span class=\"na\">href=<\/span><span class=\"s\">\"https:\/\/cdn.jsdelivr.net\/npm\/bootstrap@5.2.3\/dist\/css\/bootstrap.min.css\"<\/span> <span class=\"na\">rel=<\/span><span class=\"s\">\"stylesheet\"<\/span> <span class=\"na\">integrity=<\/span><span class=\"s\">\"sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65\"<\/span> <span class=\"na\">crossorigin=<\/span><span class=\"s\">\"anonymous\"<\/span><span class=\"nt\">&gt;<\/span>\n  <span class=\"nt\">&lt;\/head&gt;<\/span>\n  <span class=\"nt\">&lt;body&gt;<\/span>\n    <span class=\"nt\">&lt;div<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"container justify-content-center mx-auto\"<\/span><span class=\"nt\">&gt;<\/span>\n        <span class=\"nt\">&lt;h1<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"text-center\"<\/span><span class=\"nt\">&gt;<\/span>Seo Audit Tools<span class=\"nt\">&lt;\/h1&gt;<\/span>\n        <span class=\"nt\">&lt;form<\/span> <span class=\"na\">action=<\/span><span class=\"s\">\"\"<\/span> <span class=\"na\">method=<\/span><span class=\"s\">\"post\"<\/span><span class=\"nt\">&gt;<\/span>\n            <span class=\"nt\">&lt;div<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"input-group\"<\/span><span class=\"nt\">&gt;<\/span>\n                <span class=\"nt\">&lt;input<\/span> <span class=\"na\">name=<\/span><span class=\"s\">\"website\"<\/span> <span class=\"na\">type=<\/span><span class=\"s\">\"search\"<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"form-control rounded\"<\/span> <span class=\"na\">placeholder=<\/span><span class=\"s\">\"example.com\"<\/span> <span class=\"na\">aria-label=<\/span><span class=\"s\">\"Search\"<\/span> <span class=\"na\">aria-describedby=<\/span><span class=\"s\">\"search-addon\"<\/span> <span class=\"nt\">\/&gt;<\/span>\n                <span class=\"nt\">&lt;button<\/span> <span class=\"na\">type=<\/span><span class=\"s\">\"submit\"<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"btn btn-warning text-white\"<\/span><span class=\"nt\">&gt;<\/span>Audit<span class=\"nt\">&lt;\/button&gt;<\/span>\n            <span class=\"nt\">&lt;\/div&gt;<\/span>\n        <span class=\"nt\">&lt;\/form&gt;<\/span>\n    <span class=\"nt\">&lt;\/div&gt;<\/span>\n    <span class=\"nt\">&lt;div<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"container\"<\/span><span class=\"nt\">&gt;<\/span>\n      <span class=\"nt\">&lt;table<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"table\"<\/span><span class=\"nt\">&gt;<\/span>\n\n        <span class=\"nt\">&lt;thead&gt;<\/span>\n          <span class=\"nt\">&lt;tr&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"col\"<\/span><span class=\"nt\">&gt;<\/span>No.<span class=\"nt\">&lt;\/th&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"col\"<\/span><span class=\"nt\">&gt;<\/span>Word Count<span class=\"nt\">&lt;\/th&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"col\"<\/span><span class=\"nt\">&gt;<\/span>Page Title<span class=\"nt\">&lt;\/th&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"col\"<\/span><span class=\"nt\">&gt;<\/span>Meta Description<span class=\"nt\">&lt;\/th&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"col\"<\/span><span class=\"nt\">&gt;<\/span>Details Page<span class=\"nt\">&lt;\/th&gt;<\/span>\n          <span class=\"nt\">&lt;\/tr&gt;<\/span>\n        <span class=\"nt\">&lt;\/thead&gt;<\/span>\n        <span class=\"nt\">&lt;tbody&gt;<\/span>\n          {% for data in datas %}\n          <span class=\"nt\">&lt;tr&gt;<\/span>\n            <span class=\"nt\">&lt;th<\/span> <span class=\"na\">scope=<\/span><span class=\"s\">\"row\"<\/span><span class=\"nt\">&gt;<\/span>{{ loop.index }}<span class=\"nt\">&lt;\/th&gt;<\/span>\n            <span class=\"nt\">&lt;td&gt;<\/span>{{ data['word count'] }}<span class=\"nt\">&lt;\/td&gt;<\/span>\n            <span class=\"nt\">&lt;td&gt;<\/span>{{ data['page title'] }}<span class=\"nt\">&lt;\/td&gt;<\/span>\n            <span class=\"nt\">&lt;td&gt;<\/span>{{ data['meta discription'] }}<span class=\"nt\">&lt;\/td&gt;<\/span>\n            <span class=\"nt\">&lt;td&gt;&lt;a<\/span> <span class=\"na\">class=<\/span><span class=\"s\">\"btn btn-outline-primary btn-sm\"<\/span> <span class=\"na\">href=<\/span><span class=\"s\">\"\"<\/span><span class=\"nt\">&gt;<\/span>See Details<span class=\"nt\">&lt;\/a&gt;&lt;\/td&gt;<\/span>\n\n          <span class=\"nt\">&lt;\/tr&gt;<\/span>\n        <span class=\"nt\">&lt;\/tbody&gt;<\/span>\n          {% endfor %}\n      <span class=\"nt\">&lt;\/table&gt;<\/span>\n    <span class=\"nt\">&lt;\/div&gt;<\/span>\n    <span class=\"nt\">&lt;script <\/span><span class=\"na\">src=<\/span><span class=\"s\">\"https:\/\/cdn.jsdelivr.net\/npm\/bootstrap@5.2.3\/dist\/js\/bootstrap.bundle.min.js\"<\/span> <span class=\"na\">integrity=<\/span><span class=\"s\">\"sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK\/7HAuoJl+0I4\"<\/span> <span class=\"na\">crossorigin=<\/span><span class=\"s\">\"anonymous\"<\/span><span class=\"nt\">&gt;&lt;\/script&gt;<\/span>\n  <span class=\"nt\">&lt;\/body&gt;<\/span>\n<span class=\"nt\">&lt;\/html&gt;<\/span>\n<\/code><\/pre>\n<div class=\"highlight__panel js-actions-panel\">\n<div class=\"highlight__panel-action js-fullscreen-code-action\">\n    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"><title>\u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\/>\n<\/svg><\/p>\n<p>    <svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"><title>\u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    <path d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\/>\n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n<p><\/p>\n<h2><span class=\"ez-toc-section\" id=\"%D9%86%D8%AA%DB%8C%D8%AC%D9%87\"><\/span>\n<p>  \u0646\u062a\u06cc\u062c\u0647<br \/>\n<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u0627\u06cc\u0646 \u067e\u0633\u062a \u062f\u0631 \u0645\u0648\u0631\u062f \u0646\u062d\u0648\u0647 \u0627\u06cc\u062c\u0627\u062f \u0645\u0627\u0698\u0648\u0644 \u0647\u0627\u06cc \u0633\u0641\u0627\u0631\u0634\u06cc \u0628\u0631 \u0627\u0633\u0627\u0633 \u0645\u0627\u0698\u0648\u0644 \u0634\u062e\u0635 \u062b\u0627\u0644\u062b \u062f\u0631 \u067e\u0627\u06cc\u062a\u0648\u0646 \u0628\u062d\u062b \u0645\u06cc \u06a9\u0646\u062f\u060c \u0627\u06cc\u0646 \u067e\u0631\u0648\u0698\u0647 \u062f\u0631 docker \u0646\u06cc\u0632 \u067e\u0634\u062a\u06cc\u0628\u0627\u0646\u06cc \u0645\u06cc \u0634\u0648\u062f\u060c \u0644\u0637\u0641\u0627\u064b \u0622\u0646 \u0631\u0627 \u062f\u0631 github \u0628\u0631\u0631\u0633\u06cc \u06a9\u0646\u06cc\u062f\u060c \u0628\u0631\u0627\u06cc \u067e\u0631\u0648\u0698\u0647 \u0634\u06af\u0641\u062a \u0627\u0646\u06af\u06cc\u0632\u062a\u0631\u060c \u0645\u0648\u0641\u0642 \u0628\u0627\u0634\u06cc\u062f!<\/p>\n<\/p><\/div>\n","protected":false},"excerpt":{"rendered":"<p>\u0632\u0645\u06cc\u0646\u0647 \u0647\u0646\u06af\u0627\u0645\u06cc \u06a9\u0647 \u0645\u0646 \u06cc\u06a9 \u0648\u0628 \u0633\u0627\u06cc\u062a \u0631\u0627 \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0645\u06cc \u06a9\u0646\u0645 \u062a\u0627 \u0628\u0631\u062e\u06cc \u0627\u0632 \u06af\u0632\u0627\u0631\u0634 \u0647\u0627\u06cc \u062f\u0627\u062f\u0647 \u0627\u0632 \u06cc\u06a9 \u0647\u062f\u0641 \u0648\u0628 \u0631\u0627 \u062f\u0631\u06cc\u0627\u0641\u062a \u06a9\u0646\u0645\u060c \u0628\u0627\u06cc\u062f \u06cc\u06a9 \u0627\u0633\u06a9\u0631\u0627\u067e\u0631 \u0628\u0644\u0627\u062f\u0631\u0646\u06af \u0627\u06cc\u062c\u0627\u062f \u06a9\u0646\u0645 \u06a9\u0647 \u0628\u0631\u0627\u06cc \u062a\u062c\u0632\u06cc\u0647 \u0648 \u062a\u062d\u0644\u06cc\u0644 \u0648 \u067e\u06cc\u0634 \u0628\u06cc\u0646\u06cc \u06a9\u0644\u0645\u0627\u062a \u06a9\u0644\u06cc\u062f\u06cc \u0648 \u0634\u0645\u0627\u0631\u0634 \u06a9\u0644\u0645\u0647 \u0627\u0632 \u06cc\u06a9 \u0647\u062f\u0641 \u0648\u0628 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0645\u06cc \u0634\u0648\u062f\u060c \u0628\u0646\u0627\u0628\u0631\u0627\u06cc\u0646 \u0686\u0646\u062f \u0627\u0628\u0632\u0627\u0631 &hellip;<\/p>\n","protected":false},"author":2,"featured_media":5088,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"fifu_image_url":"","fifu_image_alt":"","footnotes":""},"categories":[339],"tags":[],"class_list":["post-5087","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-dev"],"_links":{"self":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts\/5087","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/comments?post=5087"}],"version-history":[{"count":0,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts\/5087\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/media\/5088"}],"wp:attachment":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/media?parent=5087"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/categories?post=5087"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/tags?post=5087"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}