파이썬으로 여러 웹 페이지 긁기

Question

from bs4 import BeautifulSoup 
import urllib, time 
class scrap(object): 
    def __init__(self): 
     self.urls = ['https://www.onthemarket.com/for-sale/property/wigan/', 'https://www.onthemarket.com/for-sale/property/wigan/?page=1', 'https://www.onthemarket.com/for-sale/property/wigan/?page=2', 'https://www.onthemarket.com/for-sale/property/wigan/?page=3', 'https://www.onthemarket.com/for-sale/property/wigan/?page=4', 'https://www.onthemarket.com/for-sale/property/wigan/?page=6'] 
     self.telephones = [] 
    def extract_info(self): 
     for link in self.urls: 
      data = urllib.request.urlopen(link).read() 
      soup = BeautifulSoup(data, "lxml") 
      for tel in soup.findAll("span", {"class":"call"}): 
       self.telephones.append(tel.text.strip()) 
      time.sleep(1) 
     return self.telephones 

to = scrap() 
print(to.extract_info()) 

무엇이 잘못 되었나요? 이 코드는 두 번째 웹 사이트 뒤에 매달려 있습니다. 목록의 각 웹 페이지에서 전화 번호를 추출해야합니다. self.urls

Answer 1

검색어 매개 변수에 headers을 입력하고 이동하면됩니다. 이것을 시도하십시오 :

from bs4 import BeautifulSoup 
import requests, time 

class scrape(object): 

    def __init__(self): 
     self.urls = ['https://www.onthemarket.com/for-sale/property/wigan/', 'https://www.onthemarket.com/for-sale/property/wigan/?page=1', 'https://www.onthemarket.com/for-sale/property/wigan/?page=2', 'https://www.onthemarket.com/for-sale/property/wigan/?page=3', 'https://www.onthemarket.com/for-sale/property/wigan/?page=4', 'https://www.onthemarket.com/for-sale/property/wigan/?page=6'] 
     self.telephones = [] 

    def extract_info(self): 
     for link in self.urls: 
      data = requests.get(link,headers={"User-Agent":"Mozilla/5.0"}) #it should do the trick 
      soup = BeautifulSoup(data.text, "lxml") 
      for tel in soup.find_all("span",{"class":"call"}): 
       self.telephones.append(tel.text.strip()) 
      time.sleep(1) 
     return self.telephones 

crawl = scrape() 
print(crawl.extract_info())