Download the URL of Python 3 in all ways

Time:2020-10-15

Get the information of the home page elements:

Target test_ URL: http://www.xxx.com.cn/

First of all, check the element. Under the a tag, we need to crawl to get the link. By getting the link path, we can locate the information we need


soup = Bs4(reaponse.text, "lxml")
urls_li = soup.select("#mainmenu_top > div > div > ul > li")

Get the URL link on the home page:

Complete the URL link of the home page. The specific code is as follows:

'''
Do you have a problem you don't understand? Python learning exchange group: 821460695 to meet your needs, information has been uploaded group files, you can download!
'''
def get_first_url():
  list_href = []
  reaponse = requests.get("http://www.xxx.com.cn", headers=headers)
  soup = Bs4(reaponse.text, "lxml")
  urls_li = soup.select("#mainmenu_top > div > div > ul > li")
  for url_li in urls_li:
    urls = url_li.select("a")
    for url in urls:
      url_href = url.get("href")
      list_href.append(head_url+url_href)
      out_url = list(set(list_href))
  for reg in out_url:
    print(reg)

The result returned by traversing the first time:

On the basis of getting the URL from the second step, traverse each page to get the URL link in the page and filter out the unnecessary information

The specific codes are as follows:


def get_next_url(urllist):
  url_list = []
  for url in urllist:
    response = requests.get(url,headers=headers)
    soup = Bs4(response.text,"lxml")
    urls = soup.find_all("a")
    if urls:
      for url2 in urls:
        url2_1 = url2.get("href")
        if url2_1:
          if url2_1[0] == "/":
            url2_1 = head_url + url2_1
            url_list.append(url2_1)
            if url2_1[0:24] == "http://www.xxx.com.cn":
              url2_1 = url2_1
              url_list.append(url2_1)
            else:
              pass
          else:
            pass
        else:
          pass
    else:
      pass
  url_list2 = set(url_list)
  for url_ in url_list2:
    res = requests.get(url_)
    if res.status_code ==200:
      print(url_)
  print(len(url_list2))

Recursive loop traversal:

Recursively crawls all the URLs, and gets_ next_ The URL () function calls itself. The code is as follows:


get_next_url(url_list2)

All codes are as follows:


import requests
from bs4 import BeautifulSoup as Bs4

head_url = "http://www.xxx.com.cn"
headers = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
}
def get_first_url():
  list_href = []
  reaponse = requests.get(head_url, headers=headers)
  soup = Bs4(reaponse.text, "lxml")
  urls_li = soup.select("#mainmenu_top > div > div > ul > li")
  for url_li in urls_li:
    urls = url_li.select("a")
    for url in urls:
      url_href = url.get("href")
      list_href.append(head_url+url_href)
      out_url = list(set(list_href))
  return out_url


def get_next_url(urllist):
  url_list = []
  for url in urllist:
    response = requests.get(url,headers=headers)
    soup = Bs4(response.text,"lxml")
    urls = soup.find_all("a")
    if urls:
      for url2 in urls:
        url2_1 = url2.get("href")
        if url2_1:
          if url2_1[0] == "/":
            url2_1 = head_url + url2_1
            url_list.append(url2_1)
            if url2_1[0:24] == "http://www.xxx.com.cn":
              url2_1 = url2_1
              url_list.append(url2_1)
            else:
              pass
          else:
            pass
        else:
          pass
    else:
      pass
  url_list2 = set(url_list)
  for url_ in url_list2:
    res = requests.get(url_)
    if res.status_code ==200:
      print(url_)
  print(len(url_list2))
  get_next_url(url_list2)


if __name__ == "__main__":
  urllist = get_first_url()
  get_next_url(urllist)

The way to crawl all the URLs under the website by Python 3 is to share all the content with you. I hope to give you a reference, and I hope you can support developpaer more.