ساخت کراولر URL برای نگاشت یک وب سایت با پایتون — راهنمای کاربردی

1# a queue of urls to be crawled next
2new_urls = deque([url])

1# a set of urls that we have already processed 
2processed_urls = set()

1# a set of domains inside the target website
2local_urls = set()
3# a set of domains outside the target website
4foreign_urls = set()
5# a set of broken urls
6broken_urls = set()

1# process urls one by one until we exhaust the queue
2while len(new_urls):
3    # move url from the queue to processed url set
4    url = new_urls.popleft()
5    processed_urls.add(url)
6    # print the current url
7    print(“Processing %s” % url)

1try:
2    response = requests.get(url)
3except(requests.exceptions.MissingSchema, requests.exceptions.ConnectionError, requests.exceptions.InvalidURL, requests.exceptions.InvalidSchema):
4    # add broken urls to it’s own set, then continue
5    broken_urls.add(url)
6    continue

1# extract base url to resolve relative links
2parts = urlsplit(url)
3base = “{0.netloc}”.format(parts)
4strip_base = base.replace(“www.”, “”)
5base_url = “{0.scheme}://{0.netloc}”.format(parts)
6path = url[:url.rfind(‘/’)+1] if ‘/’ in parts.path else url

1soup = BeautifulSoup(response.text, “lxml”)

1for link in soup.find_all(‘a’):
2    # extract link url from the anchor
3    anchor = link.attrs[“href”] if “href” in link.attrs else ‘’
4if anchor.startswith(‘/’):
5        local_link = base_url + anchor
6        local_urls.add(local_link)
7    elif strip_base in anchor:
8        local_urls.add(anchor)
9    elif not anchor.startswith(‘http’):
10        local_link = path + anchor
11        local_urls.add(local_link)
12    else:
13        foreign_urls.add(anchor)

1for i in local_urls:
2    if not i in new_urls and not i in processed_urls:
3        new_urls.append(i)

1if not link in new_urls and not link in processed_urls:
2    new_urls.append(link)

1from bs4 import BeautifulSoup
2import requests
3import requests.exceptions
4from urllib.parse import urlsplit
5from urllib.parse import urlparse
6from collections import deque
7import re
8
9url = "https://scrapethissite.com"
10# a queue of urls to be crawled
11new_urls = deque([url])
12
13# a set of urls that we have already been processed 
14processed_urls = set()
15# a set of domains inside the target website
16local_urls = set()
17# a set of domains outside the target website
18foreign_urls = set()
19# a set of broken urls
20broken_urls = set()
21
22# process urls one by one until we exhaust the queue
23while len(new_urls):
24    # move next url from the queue to the set of processed urls
25    url = new_urls.popleft()
26    processed_urls.add(url)
27    # get url's content
28    print("Processing %s" % url)
29    try:
30        response = requests.get(url)
31    except (requests.exceptions.MissingSchema, requests.exceptions.ConnectionError, requests.exceptions.InvalidURL, requests.exceptions.InvalidSchema):
32        # add broken urls to it's own set, then continue
33        broken_urls.add(url)
34        continue
35    
36    # extract base url to resolve relative links
37    parts = urlsplit(url)
38    base = "{0.netloc}".format(parts)
39    strip_base = base.replace("www.", "")
40    base_url = "{0.scheme}://{0.netloc}".format(parts)
41    path = url[:url.rfind('/')+1] if '/' in parts.path else url
42
43    # create a beutiful soup for the html document
44    soup = BeautifulSoup(response.text, "lxml")
45
46    for link in soup.find_all('a'):
47        # extract link url from the anchor
48        anchor = link.attrs["href"] if "href" in link.attrs else ''
49
50        if anchor.startswith('/'):
51            local_link = base_url + anchor
52            local_urls.add(local_link)
53        elif strip_base in anchor:
54            local_urls.add(anchor)
55        elif not anchor.startswith('http'):
56            local_link = path + anchor
57            local_urls.add(local_link)
58        else:
59            foreign_urls.add(anchor)
60
61        for i in local_urls:
62            if not i in new_urls and not i in processed_urls:
63                new_urls.append(i)
64
65print(processed_urls)