위키 피 디아 테이블에서 URL 추출하기

Question

아래 코드를 사용하여 here에서 가져온 위키 백과 테이블을 다 쳤습니다. 시험 정보 외에도 각 인용/국가 별 하이퍼 링크를 방문하여 각 페이지의 텍스트를 복사하고 싶습니다. BeautifulSoup으로 해결할 수 있습니까?

# -*- coding: utf-8 -*- 
""" 
Scrape a table from wikipedia using python. Allows for cells spanning multiple rows and/or columns. Outputs csv files for 
each table 
""" 

from bs4 import BeautifulSoup 
import urllib.request 
import os 
import codecs 

# wiki = "https://en.wikipedia.org/wiki/International_Phonetic_Alphabet_chart_for_English_dialects" 
wiki = 'https://en.wikipedia.org/wiki/List_of_national_capitals_in_alphabetical_order' 
header = {'User-Agent': 'Mozilla/5.0'} # Needed to prevent 403 error on Wikipedia 
req = urllib.request.Request(wiki, headers=header) 
page = urllib.request.urlopen(req) 
soup = BeautifulSoup(page, "html.parser") 

tables = soup.findAll("table", {"class": "wikitable"}) 

# show tables 
# for table in tables: 
#  print("###############") 
#  print(table)#.text)#[:100]) 

for tn in range(len(tables)): 
    table = tables[tn] 

    # preinit list of lists 
    rows = table.findAll("tr") 
    row_lengths = [len(r.findAll(['th', 'td'])) for r in rows] 
    ncols = max(row_lengths) 
    nrows = len(rows) 
    data = [] 
    for i in range(nrows): 
     rowD = [] 
     for j in range(ncols): 
      rowD.append('') 
     data.append(rowD) 

    # process html 
    for i in range(len(rows)): 
     row = rows[i] 
     rowD = [] 
     cells = row.findAll(["td", "th"]) 
     for j in range(len(cells)): 
      cell = cells[j] 

      # lots of cells span cols and rows so lets deal with that 
      cspan = int(cell.get('colspan', 1)) 
      rspan = int(cell.get('rowspan', 1)) 
      for k in range(rspan): 
       for l in range(cspan): 
        data[i + k][j + l] += cell.text 

     data.append(rowD) 

     # write data out 

     page = os.path.split(wiki)[1] 
    fname = 'output_{}_t{}.csv'.format(page, tn) 
    f = codecs.open(fname, 'w') # ,encoding='utf-8') 
    for i in range(nrows): 
     rowStr = ','.join(data[i]) 
     rowStr = rowStr.replace('\n', '') 
     # print(rowStr) 
     rowStr = rowStr # .encode('unicode_escape') 
     f.write(rowStr + '\n') 

    f.close() 

Answer 1

from bs4 import BeautifulSoup 
import requests 
wiki_url = 'https://en.wikipedia.org/wiki/List_of_national_capitals_in_alphabetical_order' 
print('Fetching main wiki article: %s' % wiki_url) 
page = requests.get(wiki_url).text 
print('Done. Extracting table links..') 
html = BeautifulSoup(page) 
table = html.find('table', 'wikitable') 
links = table.findAll('a') 
links_content = {} 
print('Done extracting links. About to fetch: %s links..' % len(links)) 
for link in links: 
    print('Fetching: %s' % link) 
    links_content[link] = requests.get(link).text