---
# System prepended metadata

title: 網路爬蟲
tags: [YTP共學筆記]

---

###### tags: `YTP共學筆記`
# 網路爬蟲

## 環境設置
```python=
(terminal)
pip install beautifulsoup4
pip install requests
```
## 功能
### GET
得到網頁內容
``` python=
from cgitb import text
import requests
res = requests.get("https://www.books.com.tw/?gclid=Cj0KCQjwjvaYBhDlARIsAO8PkE3xss8aBx9yAj2ywe8dnndF7umj3f9renw2DwDp6TeGnutp-HMEjCIaAhC2EALw_wcB")
print(res.text)
```
找出特定網頁內容
```python=
from bs4 import BeautifulSoup
html_sample = '\
<html> \
<body>  \
    <h1 id="title">Hello world</h1> \
    <a href="#" class="link">This is link1</a> \
    <a href="# link2" class="link">This is link2</a>\
    </body> \
    </html>'
soup = BeautifulSoup(html_sample)
print(soup.select('html')[0])
```
```python=
from cgitb import text
import requests
from bs4 import BeautifulSoup
res = requests.get("https://www.books.com.tw/?gclid=Cj0KCQjwjvaYBhDlARIsAO8PkE3xss8aBx9yAj2ywe8dnndF7umj3f9renw2DwDp6TeGnutp-HMEjCIaAhC2EALw_wcB")
soup = BeautifulSoup(res.text)
for item in soup.select('.item'):
    print(item.select('strong')[0].text,item.select('')) 
```
```python=
## books
import shutil
import requests
import sqlite3 as lite
from bs4 import BeautifulSoup
res = requests.get("https://www.books.com.tw/products/0010610509?sloc=main")
image = requests.get("https://www.books.com.tw/img/001/061/05/0010610509.jpg")
soup = BeautifulSoup(res.text)
print(soup)
print(soup.prettify())
tmp = soup.find("meta")
print(tmp.find_parent("meta"))
print(soup.find_all("meta"))
print(soup.find_parents("a"))

print(soup.find_all(["meta","meta"]))
print(soup.find_all("a",class_="trace_box"))
for meta in soup.select("head"):
    print(meta.text)
print(soup.find("title").text)
f = open('image.png','wb')
shutil.copyfileobj(res.raw,f)

f = open('image.png','wb')
shutil.copyfileobj(res.raw,f)
print(title.select_one("a").get("href"))
print(title.select_one("a").getText())
```
### POST
```python=
import requests
payload = {
'parameter1' = 'your parameter1'
'parameter2' = 'your parameter2'
.
.
.
'parameterN' = 'your parameterN'
res=requests.post('the request url' ,data=payload)
print(res.text)
}
```
