Package
與網路爬蟲

Vincent55 楊竣鴻

`whoami`

Package 與網路爬蟲 Vincent55 楊竣鴻 2023-07-22 09:00 - 12:00

	<head>
	<link rel="stylesheet" type="text/css" href="mycss.css">
	<style>
	body{
	background:#fff;
	color:#777;
	}
	h1 {
	font-weight:bold;
	font-style:italic;
	font-family:sans-serif;
	color:green;
	}
	a {
	color: #0077cc;
	text-decoration: none;
	}


	.container {
	border: 2px solid #ccc;
	padding: 20px;
	border-radius: 10px;
	}

	</style>
	</head>

	import requests
	#先將欲發出 GET 請求的網址先存在 url
	url = 'https://example.com/'
	#對 url 發出 GET 請求，並將 Response 物件存在 res
	res = requests.get(url)

	print(type(res), res)

	#Output: <class 'requests.models.Response'> <Response [200]>

	import requests
	#先將欲發出 GET 請求的網址先存在 url
	url = 'https://example.com/'
	#對 url 發出 GET 請求，並將 Response 物件存在 res
	res = requests.get(url)
	print(res.text)
	'''
	<!doctype html>
	<html>
	<head>
	<title>Example Domain</title>

	<meta charset="utf-8" />
	<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1" />
	<style type="text/css">
	body {
	background-color: #f0f0f2;
	margin: 0;
	padding: 0;
	font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;

	}
	div {
	width: 600px;
	margin: 5em auto;
	padding: 2em;
	background-color: #fdfdff;
	border-radius: 0.5em;
	box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
	}
	a:link, a:visited {
	color: #38488f;
	text-decoration: none;
	}
	@media (max-width: 700px) {
	div {
	margin: 0 auto;
	width: auto;
	}
	}
	</style>
	</head>

	<body>
	<div>
	<h1>Example Domain</h1>
	<p>This domain is for use in illustrative examples in documents. You may use this
	domain in literature without prior coordination or asking for permission.</p>
	<p><a href="https://www.iana.org/domains/example">More information...</a></p>
	</div>
	</body>
	</html>
	'''

	import requests
	import json
	#先將欲發出 GET 請求的網址先存在 url
	url = 'https://httpbin.org/json'
	#對 url 發出 GET 請求，並將 Response 物件透過 json 解析後存在 res
	res = requests.get(url).json()
	print(type(res), res)
	print(res['slideshow']['slides'][1]['title'])

	# 將回傳值存於 result.json
	with open('result.json', 'w', encoding='utf-8') as f:
	json.dump(res, f, indent=2, sort_keys=True, ensure_ascii=False)

	'''
	<class 'dict'> {'slideshow': {'author': 'Yours Truly', 'date': 'date of publication', 'slides': [{'title': 'Wake up to WonderWidgets!', 'type': 'all'}, {'items': ['Why <em>WonderWidgets</em> are great', 'Who <em>buys</em> WonderWidgets'], 'title': 'Overview', 'type': 'all'}], 'title': 'Sample Slide Show'}}
	Overview
	'''

	from bs4 import BeautifulSoup
	html_text = """
	<html><head></head><body><h1>Hello, World!</h1></body></html>
	"""
	soup = BeautifulSoup(html_text, "html.parser")
	print(soup.prettify())

	from bs4 import BeautifulSoup
	import requests
	# 將 resp.text 也就是原始碼字串資料放到 BeautifulSoup 第一個參數內，並用 html.parser 當作第二個參數解析 HTML 內容
	soup = BeautifulSoup(requests.get(url).text, "html.parser")

	# 網頁的 title
	print(soup.title.getText())

	# 第一個尋找到的 p 標籤的文字
	print(soup.find('p').getText())

	# 第一個尋找到 class 為 magic 的標籤
	print(soup.find(class_="magic"))

	# a 標籤的 href 屬性
	print(soup.find('a')['href'])

	# 全部 class 為 article 的標籤的文字
	articles = soup.find_all(class_="article")
	for article in articles:
	print(article.getText())

	from bs4 import BeautifulSoup
	import requests
	url = "https://www.ptt.cc/bbs/Baseball/index.html"
	r = requests.get(url)
	soup = BeautifulSoup(r.text, "html.parser")
	for article in soup.find_all(class_="r-ent"):
	print(article)