---
# System prepended metadata

title: SC_0813

---

# SC_0813

## `demo1_env_check_warm_up.py`
```python
import os, sys
import numpy
import pandas
import matplotlib

# ctrl+shift+F10
print(os.getcwd())
print(sys.executable)
print(numpy.__version__)
print(pandas.__version__)
print(matplotlib.__version__)
# ctrl+alt+L
# alt+Enter
demoObj = {'course': "Data Processing", 'duration': 28, 'instructor': "MarkHo"}

```

## `demo2_arithmatic.py`
```python

import numpy as np

print("hello" + ",world")
l1 = [1, 2, 3]
l2 = [4, 5, 6]
print(l1 + l2)
a1 = np.array(l1)
a2 = np.array(l2)
print(a1 + a2, a1 - a2, a1 * a2, a1 / a2)

print(a2 / a1, a2 // a1, a2 % a1)
print(a1 ** a2, a2 ** a1)



```

## `demo3_complex.py`
```python
c1 = 2 + 3j
c2 = 5
c3 = 4j
c4 = -5
print(type(c1), type(c2), type(c3))
print(c1.conjugate())
print(c1*5, c1*5j)
print(abs(c1), abs(c2), abs(c3), abs(c4), (2**2+3**2)**0.5)
print(c1.real, c1.imag)

```

## `demo4_fraction.py`
```Python
from fractions import Fraction

print(Fraction(250, 72))
print(Fraction(5, 2) + Fraction(19, 6) + Fraction(27, 3))
frac1 = Fraction(250, 70)
print(frac1.denominator, frac1.numerator)
print(f"除數是{frac1.denominator}, 被除數是{frac1.numerator}")

```

## `demo5_decimal.py`
```python
from decimal import Decimal as Dec

print(Dec(2.968))
print(Dec('2.968'))
print(Dec(0.001)*Dec(2968)-Dec(2.968))
print(Dec('0.001')*Dec(2968)-Dec('2.968'))
```

## `demo6_object_equality.py`
```python
class Person:
    def __init__(self, age):
        self.age = age


age = 38
print(f"age id = {hex(id(age))}")
age = 39
print(f"age id = {hex(id(age))}")
v1 = 39
print(f"v1 id={hex(id(v1))}")
print(v1 == age, v1 is age)
person1 = Person(38)
print(f"person1 id={hex(id(person1))}")
person2 = person1
print(f"person2 id={hex(id(person2))}")
person3 = Person(38)
print(f"person3 id={hex(id(person3))}")
person1.age = 39
person3.age = 39
print(f"person1 id={hex(id(person1))}")
print(f"person2 id={hex(id(person2))}")
print(f"person3 id={hex(id(person3))}")


```
## `demo7_equal_identical.py`
```python
l1 = ['apple', 'banana']
l2 = l1
l3 = ['apple', 'banana']
print(l1 is l2, l1 == l2)
print(l1 is l3, l1 == l3)

```

## `demo8_fp1.py`
```python
from pprint import pprint
courses = [{'name': 'poop', 'field': 'python', 'attendee': 10, 'remote': False},
           {'name': 'bdpy', 'field': 'python', 'attendee': 15, 'remote': True},
           {'name': 'andbiz3', 'field': 'android', 'attendee': 5, 'remote': False}]
pprint(courses)
courses = [{'name': 'poop', 'field': 'python', 'attendee': 10, 'remote': False},
           {'name': 'bdpy', 'field': 'python', 'attendee': 15, 'remote': True},
           {'name': 'andbiz3', 'field': 'android', 'attendeen': 5, 'remote': False}]
for course in courses:
    print(course['attendee'])



```

## `demo9_fp2.py`
```python

import collections

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
print(poop)
print(poop.name, poop.field, poop.attendee)
poop.name='aiocv'

```
## `demo10_fp3.py`
```python
import collections
from pprint import pprint
course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
courses = [poop, bdpy, andbiz]
pprint(courses)
del courses[0]
pprint(courses)

```

## `demo11_fp4.py`
```python=
import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)

# using filter
result1 = filter(lambda x: x.remote is True, courses)
print(type(result1))
pprint([x for x in result1])

result2 = filter(lambda x: x.attendee >= 10, courses)
print(next(result2))
print(next(result2))

result3 = filter(lambda x: x.remote is True, courses)
result4 = filter(lambda x: x.attendee > 10, result3)
pprint([x for x in result4])



```

## `demo12_fp5.py`
```Python
import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
filter1 = lambda x: x.remote is True
result1 = tuple(filter(filter1, courses))
print(type(result1))
pprint(result1)

pprint(tuple(filter(filter1, courses)))

```

## `demo13_fp6.py`
```Python
import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
available1 = lambda c: c.attendee >= 8


def available(course):
    return course.attendee >= 8


def isRemote(course):
    return course.remote is True


validRemoteCourses = tuple(filter(isRemote, filter(available, courses)))
pprint(validRemoteCourses)

```

## `demo14_fp7.py`
```python

import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)

pprint([c for c in courses if c.field == 'python'])
pprint(tuple([c for c in courses if c.field == 'python']))
pprint(tuple(c for c in courses if c.field == 'python'))

```
## `demo15_fp8.py`
```python
import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
name_and_field = map(lambda x: {'name': x.name, 'field': x.field}, courses)
pprint(tuple(name_and_field))

```
## demo16_fp9.py
```python=
import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)

pprint([{'name': c.name, 'income': c.attendee * 8000} for c in courses])
pprint(tuple({'name': c.name, 'income': c.attendee * 8000} for c in courses))

```

## demo17_fp10.py
```python=
import collections
from pprint import pprint
from functools import reduce
course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
income_tuple = tuple({'name': c.name, 'income': c.attendee * 8000} for c in courses)
total_income = reduce(lambda acc, val: acc+val['income'], income_tuple, 0)
pprint(total_income)
pprint(sum(x['income'] for x in income_tuple))

```

## demo18_fp11.py
```python=
import collections
from pprint import pprint
from functools import reduce

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def reducer(acc, val):
    acc[val.field].append(val.name)
    return acc


courses_by_category = reduce(reducer, courses, {'python': [], 'android': []})
pprint(courses_by_category)
```

## demo19_fp12

```python

import collections
from functools import reduce
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
cplus = course(name='cplus', field='c++', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv,cplus)

def reducer(acc, val):
    acc.setdefault(val.field, [])
    acc[val.field].append(val.name)
    return acc
courses_by_category = reduce(reducer, courses, {})
pprint(courses_by_category)
```

## demo20_fp13
```python=
import collections
from functools import reduce
from pprint import pprint
from collections import defaultdict

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def reducer(acc, val):
    acc[val.field].append(val.name)
    return acc


courses_by_category = reduce(reducer, courses, defaultdict(list))
pprint(courses_by_category)
pprint(dict(courses_by_category))
```

## demo21_fp14
```python=
import collections
from pprint import pprint
import itertools

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
sorted_courses = sorted(courses, key=lambda x: x.field)
pprint(sorted_courses)
courses_by_remote = {c[0]: list(c[1])
                     for c in itertools.groupby(sorted_courses, lambda x: x.field)}
pprint(courses_by_remote)
courses_by_remote2 = {c[0]: list([c.name for c in c[1]])
                      for c in itertools.groupby(sorted_courses, lambda x: x.field)}
pprint(courses_by_remote2)
courses_by_remote3 = [(c[0], list(c.name for c in c[1]))
                      for c in itertools.groupby(sorted_courses, lambda x: x.field)]
pprint(courses_by_remote3)

```

## demo22_fp15
```python=
from functools import reduce

print({'a': 0, 'b': 1, 'a': 0 + 1, 'b': 1 + 2})

import collections
from pprint import pprint

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)
reducer = lambda acc, val: {**acc, **{val.field: acc[val.field] + [val.name]}}
courses_by_category = reduce(reducer, courses, {'python': [], 'android': []})
pprint(courses_by_category)


```

## demo23_fp16.py
```python=
import collections
from pprint import pprint
import time

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def transform(x):
    print(f"process record:{x.name}")
    time.sleep(3)
    result = {'name': x.name, 'revenue': x.attendee * 5000}
    print(f"done process record:{x.name}")
    return result

start = time.time()
result = tuple(map(transform, courses))
end = time.time()

print(f"total time:{end-start:.2f} seconds")
pprint(result)


```

## demo24_fp17
```python=
import collections
from pprint import pprint
import time
import multiprocessing
course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)
import os
poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def transform(x):
    print(f"process:{os.getpid()} record:{x.name}")
    time.sleep(3)
    result = {'name': x.name, 'revenue': x.attendee * 5000}
    print(f"done process:{os.getpid()} record:{x.name}")
    return result

if __name__ == '__main__':
    start = time.time()
    pool = multiprocessing.Pool(processes=4, maxtasksperchild=1)
    print(f"now we run with process:{pool._processes}")
    result = pool.map(transform, courses)
    end = time.time()

    print(f"total time:{end-start:.2f} seconds")
    pprint(result)



```

## demo25_fp18

```python=
import collections
from pprint import pprint
import time
import concurrent.futures

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)
import os

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def transform(x):
    print(f"process:{os.getpid()} record:{x.name}")
    time.sleep(3)
    result = {'name': x.name, 'revenue': x.attendee * 5000}
    print(f"done process:{os.getpid()} record:{x.name}")
    return result


if __name__ == '__main__':
    start = time.time()
    with concurrent.futures.ProcessPoolExecutor() as executor:
        result = executor.map(transform, courses)
    end = time.time()

    print(f"total time:{end-start:.2f} seconds")
    pprint(result)


```
# demo25_fp18(change to thread)
```python=
import collections
from pprint import pprint
import time
import concurrent.futures

course = collections.namedtuple('course', ['name', 'field', 'attendee', 'remote'])
print(type(course))
print(course)
import os

poop = course(name='poop', field='python', attendee=10, remote=False)
bdpy = course(name='bdpy', field='python', attendee=15, remote=True)
pykt = course(name='python', field='python', attendee=9, remote=False)
andbiz = course(name='andbiz', field='android', attendee=18, remote=False)
aiocv = course(name='aiocv', field='python', attendee=10, remote=True)
courses = (poop, bdpy, andbiz, pykt, aiocv)


def transform(x):
    print(f"process:{os.getpid()} record:{x.name}")
    time.sleep(3)
    result = {'name': x.name, 'revenue': x.attendee * 5000}
    print(f"done process:{os.getpid()} record:{x.name}")
    return result


if __name__ == '__main__':
    start = time.time()
    with concurrent.futures.ThreadPoolExecutor() as executor:
    #with concurrent.futures.ProcessPoolExecutor() as executor:
        result = executor.map(transform, courses)
    end = time.time()

    print(f"total time:{end-start:.2f} seconds")
    pprint(result)

```

## demo26_library
```python=
def foo(a, b):
    return "[demo26][foo]result=" + str(a + b)


def bar(a, b):
    return "[demo26][bar]result=" + str(a * b)


print("inside demo26, foo(1,2)=", foo(1, 2))
print("inside demo26, bar(3,4)=", bar(3, 4))

```

## demo27_use_library
```python=

import demo26_library

print(demo26_library.foo(1, 2))
print(demo26_library.bar(3, 4))

import demo26_library as library

print(library.foo(5, 6))
print(library.bar(7, 8))

from demo26_library import foo, bar

print(foo(9, 10))
print(bar(11, 12))

from demo26_library import foo as f1, bar as f2

print(f1(13, 14))
print(f2(15, 16))

```

## demo28_variable_function_call.py
```python=
def sample_call(fix1, fix2, *args):
    print(fix1)
    print(fix2)
    print(list(args))


sample_call("hello", "world")
sample_call("hihi", "welcome", "ABC123")
sample_call("multiple", "input", 5, 300, None, 5 + 4j)
l1 = ['Sunday', 'Monday', 'Tuesday']
sample_call("pass a", "list", l1)
sample_call("pass a", "list", *l1)


```

## demo29_key_value_function_call.py
```python=
def sample_key_value_arguments(fix1, fix2, **kwargs):
    print(f"fix1={fix1}, fix2={fix2}")
    for key, value in kwargs.items():
        print(f"parameter name={key}, value={value}")


sample_key_value_arguments(500, "hi", name="mark", location="Taipei")
sample_key_value_arguments("hello", 'world')
sample_key_value_arguments("hihi", "welcome",
                           course="BDPY",
                           duration=35,
                           period=['Sun', 'Mon', 'Tue'])
course = {'name': "poop", "level": "entry", "duration": 35}
sample_key_value_arguments("unpack"," a dict", **course)


```
## demo30_fix_argument1.py
```python=
def lab30(fix1, fix2, fix3):
    print("---")
    print(f"arg1={fix1}, arg2={fix2}, arg3={fix3}")
    print("---")


args = ("two", 3.0)
lab30(1, *args)
lab30(*args, 5)
args2 = ['two', 3.0]
lab30('OK', *args2)
args3 = {'Hi', "OK", 'Welcome'}
lab30(*args3)
args4 = {'course':"PYKT",'level':'advanced',"duration":35}
lab30(*args4)


```

## dmeo31_fix_argument2
```python=
def dumpEmployee(id, name, department):
    print(f"id={id}")
    print(f"name={name}")
    print(f"department={department}")


dumpEmployee(5, "mark", "R&D")
dumpEmployee(id=5, name="Mark Ho", department="Research&Development")
dumpEmployee(5, name="Mark Lin", department="FAE")
emp1 = {'id': 1, 'name': "Kelvin", 'department': 'Sales'}
dumpEmployee(**emp1)
emp2 = (2, "James", "legal")
dumpEmployee(*emp2)

```

## demo32_str_repr.py
```python=
from datetime import datetime

now = datetime.now()
print("repr", repr(now))
print("str", str(now))
print(now)
print('inside list:', [now])
print('inside tuple', (now,))
print('not inside tuple', (now))
print('inside dict',{'k1':now})
print('inside dict, but keep str',{'k1':str(now)})

```

## demo32_system_command.py
```python=
import os
import sys
print(os.getcwd())
print(sys.executable)
print(sys.argv)
PATH1 = "logs"
os.mkdir(PATH1)
os.chdir(PATH1)
print(os.getcwd())
os.chdir("..")
os.rmdir(PATH1)


```

## demo33_math_random.py
```python=
import math
import random

print(math.pi, math.log10(10), math.log10(5))
print(math.sqrt(5))
for x in range(50):
    print(random.randint(10, 20))
stores = ['7-11', 'fami', 'Ok', "Hi-Life"]
for x in range(20):
    print(random.choice(stores))
cards = ['A', 'K', 'Q', 'J', 10]
for x in range(10):
    random.shuffle(cards)
    print(cards)

```

## demo34_itertools.py
```python=
import itertools

r1 = itertools.chain('abc', "123", 'PQRST')
t1 = tuple(r1)
print(t1)
print(t1)

TEAMS = ['A', 'B', 'C', 'D', 'E']
r2 = itertools.permutations(TEAMS, 2)
t2 = tuple(r2)
print(len(t2), t2)
r3 = itertools.combinations(TEAMS, 3)
t3 = tuple(r3)
print(len(t3), t3)

```

## `demo35_shutil.py`
```python=
import shutil
import os

ORIGINAL = "data1"
DUPLICATE = "data2"
shutil.copytree(ORIGINAL, DUPLICATE)
os.chdir(DUPLICATE)
print("now inside:", os.getcwd())
os.chdir("..")
shutil.rmtree(DUPLICATE)

```

## `demo36_json.py`
```python=
import json

v1 = ["Sunday", "monday", 100, 3.14, None, '中文', {"name": "Mark"}]
print(type(v1))
r1 = json.dumps(v1)
print(type(r1))
print(r1)
v2 = {"course": "BDPY", "instructor": "Mark",
      "period": ['Thursday', 'Friday']}
r2 = json.dumps(v2)
print(type(r2))
print(r2)

```

## `demo37_bugzilla.py`
* (https://bugzilla.mozilla.org/rest/bug/35)[https://bugzilla.mozilla.org/rest/bug/35]
* (json viewer)[https://chrome.google.com/webstore/detail/json-viewer/gbmdgpbipfallnflgajpaliibnhdgobh?utm_source=chrome-ntp-icon]
```python=

import requests

proxies = {
    "http": "10.239.9.190:443",
    "https": "10.239.9.190:443",
}

URL = 'https://bugzilla.mozilla.org/rest/bug/35'
#response = requests.get(URL, proxies=proxies, verify=False)
response = requests.get(URL, proxies=None, verify=False)
print(response.status_code)
print(type(response.json()))
result = response.json()
for k,v in result.items():
    print(f"key={k}, value={v}")
bugs = result["bugs"]
firstBug = bugs[0]
firstBugCreator = firstBug["creator"]
print(firstBugCreator)



```

## demo38_ucom_http.py
```python=
import requests
from bs4 import BeautifulSoup

proxies = {
    "http": "10.239.9.190:443",
    "https": "10.239.9.190:443",
}

URL = 'https://www.uuu.com.tw/'
# r = requests.get(URL, proxies=proxies, verify=False)
r = requests.get(URL, proxies=None, verify=False)
soup = BeautifulSoup(r.content, "html.parser")
print(type(soup))
print(soup.title.name)
print(soup.title.string)

hot_courses = soup.find('div', {'id': 'course_list'})
print(type(hot_courses))
print(hot_courses)
items = hot_courses.find_all('a')
for item in items:
    print(item)

```

## demo39_urllib_get_image
* (cwb image)[https://www.cwb.gov.tw/Data/satellite/LCC_IR1_CR_2750/LCC_IR1_CR_2750-2020-08-14-10-30.jpg]
```python=

from urllib.request import urlopen
from urllib import request
from PIL import Image

URL = 'https://www.cwb.gov.tw/Data/satellite/LCC_IR1_CR_2750/LCC_IR1_CR_2750-2020-08-14-10-30.jpg'

# for SC proxy settings
# proxy = request.ProxyHandler({'https': '10.239.9.190:443'})
# opener = request.build_opener(proxy)
# request.install_opener(opener)
# end of SC proxy setting

fileToSave = urlopen(URL)
# manual make a directory images
image = Image.open(fileToSave)
image.save('images/demo39.jpg')

# get half size
halfSize = (image.size[0] // 2, image.size[1] // 2)
halfImage = image.resize(halfSize, Image.ANTIALIAS)
halfImage.save('images/demo39_small.jpg')

rot1 = halfImage.transpose(Image.ROTATE_90)
rot1.save('images/r90.jpg')
rot2 = halfImage.transpose(Image.ROTATE_180)
rot2.save('images/r180.jpg')
rot3 = halfImage.transpose(Image.ROTATE_270)
rot3.save('images/r270.jpg')
rot4 = halfImage.rotate(60)
rot4.save('images/r60.jpg')

```

## README.md

```markdown
# Python and Data Wrangling 4 days

## `requests` under proxy usage
```python
proxies = {
    "http": "10.239.9.190:443",
    "https": "10.239.9.190:443",
}

URL = 'https://www.uuu.com.tw/'
# r = requests.get(URL, proxies=proxies, verify=False)
r = requests.get(URL, proxies=None, verify=False)
```



## `urlopen`under proxy usage
```python
# for SC proxy settings
# proxy = request.ProxyHandler({'https': '10.239.9.190:443'})
# opener = request.build_opener(proxy)
# request.install_opener(opener)
# end of SC proxy setting
```
```
```
## demo40_file_io
```txt (Python_Introduction)
Python（英國發音：/ˈpaɪθən/ 美國發音：/ˈpaɪθɑːn/）是一種廣泛使用的直譯式、進階程式、通用型程式語言，由吉多·范羅蘇姆創造，第一版釋出於1991年。Python是ABC語言的後繼者，也可以視之為一種使用傳統中綴表達式的LISP方言[4]。Python的設計哲學強調代碼的可讀性和簡潔的語法（尤其是使用空格縮排劃分代碼塊，而非使用大括號或者關鍵詞）。相比於C++或Java，Python讓開發者能夠用更少的代碼表達想法。不管是小型還是大型程式，該語言都試圖讓程式的結構清晰明瞭。

Python（パイソン）インタープリタ型の高水準汎用プログラミング言語である。グイド・ヴァン・ロッサムにより創り出され、1991年に最初にリリースされたPythonの設計哲学は、その顕著なホワイトスペースの使用によってコードの可読性が重視されている。その言語構成とオブジェクト指向のアプローチは、プログラマが小規模なプロジェクトから大規模なプロジェクトまで、明確で論理的なコードを書くのを支援することを目的としている。

Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum and first released in 1991, Python's design philosophy emphasizes code readability with its notable use of significant whitespace. Its language constructs and object-oriented approach aim to help programmers write clear, logical code for small and large-scale projects.[28]

```
```python
# mkdir data
# make a file Python_Introduction.txt

file1 = open('data/Python_Introduction.txt',
             encoding='utf8')
readme_txt = file1.read()
print(type(readme_txt))
print(readme_txt)
file1.close()

with open('data/Python_Introduction.txt',
             encoding='utf8') as file2:
    readme_txt2 = file2.read()
    print(type(readme_txt2))
    print(readme_txt2)

file3 = open('data/clone1','w', encoding='utf8')
file3.write(readme_txt)
file3.close()

with open('data/clone2','w', encoding='utf-8') as file4:
    file4.write(readme_txt2)


```

## demo41_csv_read.py
```python

import csv

sampleFile = open('data/demo41.csv', encoding='utf8')
sampleReader = csv.reader(sampleFile)
sampleData = list(sampleReader)
sampleFile.close()
print(type(sampleData))
print(sampleData)

```

## demo42_tk1.py
```python
import tkinter
from tkinter import font
import tkinter.font

counter = 10
counter2 = [10]


def callback1():
    global counter
    # counter = 0
    # print('button clicked')
    label1.config(text=f'button clicked {counter} times')
    counter += 1


def callback2():
    label2.config(text=f"button2 clicked {counter2[0]} times")
    counter2[0] += 1


def callback3():
    label1.config(text=f'button3 clicked {counter3.get()} times')
    counter3.set(counter3.get() + 1)


top = tkinter.Tk()
counter3 = tkinter.IntVar()
counter3.set(10)
# query available font
for f in font.families():
    print(f)
myFont1 = font.Font(family="Tahoma", size=24)
label1 = tkinter.Label(top, text='Hello Tk@Python',
                       font=myFont1, fg='#C0FFEE',
                       bg='#000000', padx=20, pady=10)
label2 = tkinter.Label(top, text="mark@patristar.com",
                       font=myFont1, fg='#000000',
                       bg='#FFC0EE', padx=10, pady=20)
button1 = tkinter.Button(top, text='click me1',
                         font=myFont1, fg='#C0FFEE',
                         bg='#440000', command=callback1)
button2 = tkinter.Button(top, text='click me2',
                         font=myFont1, fg='#FFEEC0',
                         bg='#004400', command=callback2)
button3 = tkinter.Button(top, text='clicked me3',
                         font=myFont1, fg='#EEFFC0',
                         bg='#000044', command=callback3)

label2.pack()
label1.pack()
button1.pack()
button2.pack()
button3.pack()
top.mainloop()






```

## demo43_tk2.py
```python
import tkinter
from tkinter import font
import tkinter.font


def function1(ev):
    label1.config(text=f'left single clicked:{ev}', bg='#880000')


def function2(ev):
    label1.config(text=f'right double clicked{ev}', bg='#008800')
def function3(ev):
    label1.config(text=f"middle click drag:{ev}", bg='#000088')

top = tkinter.Tk()
myFont1 = font.Font(family="Tahoma", size=48)
label1 = tkinter.Label(top, text="status", font=myFont1, fg='#C0FFEE', bg='#000000', padx=30, pady=20)
button1 = tkinter.Button(top, text='click', font=myFont1, fg='#000000', bg='#FFC0EE', padx=20, pady=30)
label1.pack()
button1.pack()


button1.bind('<Button-1>', function1)
button1.bind('<Double-3>', function2)
button1.bind('<B2-Motion>', function3)
top.mainloop()



```

## demo44_tk3.py
```python
import tkinter
from tkinter import font
import tkinter.font

def func1(ev):
    message1.config(text=f'move to [{ev.x},{ev.y}]')

top = tkinter.Tk()
myFont1 = font.Font(family="Tahoma", size=48)
message1 = tkinter.Message(top, text="detect mose move", font=myFont1)
label1 = tkinter.Label(top, text="detect area", bg='#C0FFEE', font=myFont1,
                       padx=30, pady=30)
message1.pack()
label1.pack()
label1.bind('<Motion>', func1)

top.mainloop()


```

## demo45_tk4.py
```python
import tkinter
from tkinter import font
import tkinter.font


def func1():
    label1.config(text="you will buy pixel5")


def func2():
    label1.config(text="you will buy iphon12")


def func3():
    if var1.get() is 1:
        label1.config(text="you will buy pixel5")
    elif var1.get() is 2:
        label1.config(text="you will buy iphon12")


top = tkinter.Tk()
myFont1 = font.Font(family="Tahoma", size=48)
var1 = tkinter.IntVar()
var1.set(2)
label1 = tkinter.Label(top, font=myFont1, text="you will choose from below:")
button1 = tkinter.Radiobutton(top, font=myFont1, text='Google', value=1,
                              variable=var1, command=func3)
button2 = tkinter.Radiobutton(top, font=myFont1, text='Apple', value=2,
                              variable=var1, command=func3)

label1.pack()
button1.pack()
button2.pack()
top.mainloop()



```

## demo46_tk5.py
```python
import tkinter
from tkinter import font
import tkinter.font


def func(scale):
    label1.config(text=formattedString % int(scale))


top = tkinter.Tk()

formattedString = 'value=%d'
value = tkinter.IntVar()
value.set(0)

myFont1 = font.Font(family="Tahoma", size=36)
label1 = tkinter.Label(top, text=formattedString % value.get(), font = myFont1)
scale = tkinter.Scale(top, label='Scale', font=myFont1, orient='h',from_=0, to=100,
                      showvalue=True, variable=value, command=func)
label1.pack()
scale.pack()
top.minsize(300,300)
top.maxsize(300,300)
top.mainloop()

```

## demo47_tk6
```python
import tkinter
from tkinter import font
import tkinter.font


def display(ev):
    label1.config(text=entry1.get())
    print(ev)


top = tkinter.Tk()

myFont1 = font.Font(family="Tahoma", size=36)
label1 = tkinter.Label(top, text="Input Text", font=myFont1)
entry1 = tkinter.Entry(top, font=myFont1)
button1 = tkinter.Button(top, text='submit', font=myFont1)
label1.pack()
entry1.pack()
button1.pack()
entry1.bind('<Return>', display)
button1.bind('<Button-1>', display)
top.mainloop()


```

## demo48
```python
import graphviz as gv

print(gv.__version__)

g1 = gv.Digraph(format='svg')

g1.node('A')
g1.node('B')
g1.edge('A', 'A')
g1.edge('A', 'A')
g1.edge('A', 'A')
g1.edge('A', 'B')
g1.edge('B', 'B')
g1.edge('B', 'B')

g1.render('graph/demo48')

```

## demo49_gv2

* download this, and unzip
* (https://www2.graphviz.org/Packages/stable/windows/10/msbuild/Release/Win32/graphviz-2.38-win32.zip)[https://www2.graphviz.org/Packages/stable/windows/10/msbuild/Release/Win32/graphviz-2.38-win32.zip]
```python

import functools
import graphviz as gv
from itertools import combinations

graph = functools.partial(gv.Graph, format='svg')
digraph = functools.partial(gv.Digraph, format='svg')
g3 = graph()
g4 = digraph()


# 'A','Apple','Google'
def add_nodes(graph, nodes):
    for n in nodes:
        if isinstance(n, tuple):
            graph.node(n[0], **n[1])
        else:
            graph.node(n)
    return graph


def add_edges(graph, edges):
    for e in edges:
        if isinstance(e[0], tuple):
            graph.edge(*e[0], **e[1])
        else:
            graph.edge(*e)
    return graph


teams = ['apple', 'google', 'facebook']
races = tuple(combinations(teams, 2))
g3 = add_edges(add_nodes(g3, teams), races)
print(g3.source)
g3.render('images/demo50_g3')

nodes = [('A', {'label': 'Apple'}),
         ('B', {'label': 'Google'}),
         ('C', {'label': '臉書'}),
         ('D', {})]
edges = [(('A', 'B'), {'label': 'phone maker compete'}),
         (('A', 'C'), {'label': 'game platform'}),
         (('B', 'C'), {'label': 'AI develop'}),
         (('B', 'D'), {})]
g4 = add_nodes(g4, nodes)
g4 = add_edges(g4, edges)
g4.render('graph/demo50_g4')

styles = {
    'graph': {
        'label': 'GAFA',
        'fontsize': '36',
        'fontcolor': '#C0FFEE',
        'bgcolor': '#99CCFF',
        'rankdir': 'BT',
        'fillcolor': '#FFC0EE'
    }
}
def apply_styles(graph, styles):
    graph.graph_attr.update(('graph' in styles and styles['graph']) or {})
    return graph

g4 = apply_styles(g4, styles)
g4.render('graph/demo50_g4_complete')

```

## demo51_and_or_explain.py
```python
x1 = True
x2 = False
x3 = False
print(x1 and x2, x1 and x1, x2 and x2)

x2s = [3.14, 500, 'hello world', 3 + 4j, None, True, False]

for x2 in x2s:
    print(x1 and x2)
print("@@@@@@@@@")
for x2 in x2s:
    print(x3 or x2)

```
## demo52_class_object_creation
```python
# ctrl+H
class MyClass(object):
    pass

inst1 = MyClass()
print(f"MyClass type:{type(MyClass)}, generate instance type:{type(inst1)}")
print(f"inst1 class = {inst1.__class__}, class name={inst1.__class__.__name__}")
print(f"inst1 bases = {inst1.__class__.__bases__}")
print(f"inst1 class equal to type? {type(inst1)==MyClass}")

class MySubClass(MyClass):
    pass
inst2 = MySubClass()
print(f"my subclass bases = {inst2.__class__.__bases__}")
```

## demo53_object_class_property
```python
class Car:
    vendor = "lexus"
    valid = True


c1 = Car()
c2 = Car()
print(c1.vendor, c2.vendor, Car.vendor)
print(c1.valid, c2.valid, Car.valid)
Car.valid = False
print(c1.valid, c2.valid, Car.valid)
c1.valid = True
print(c1.valid, c2.valid, Car.valid)
c1.capacity = 7
c2.color='RED'
print(c1.capacity, c1.valid, c1.vendor)
print(c2.color, c2.vendor, c2.valid)
Car.max = 10000
print(c1.max, c2.max, Car.max)

```

## demo54_class_instance_property
```python
class Team:
    name = 'Normal Team'
    pass


team1 = Team()
print(team1.name)

team1.name = 'Big Data Team'
team2 = Team()
print(team1.name, team2.name, Team.name, sep='||')

del team1.name
print(team1.name, team2.name, Team.name, sep='||')




```

## demo55_whole_class
```python
class Team:
    member = 7  # class variable

    def working_hour(self):  # method
        return self.day

    def all_working_hour(self):  # method
        self.day = 7  # field
        return self.day * self.member

    @classmethod
    def get_member(cls):
        return cls.member

    @staticmethod
    def calculate(x, y):
        return x ** y + 200


print(Team.calculate(3, 5))
print(Team.get_member())
t1 = Team()
print(t1.all_working_hour())
print(t1.working_hour())
print(Team.get_member(), t1.get_member())

```

## demo56_field_practice.py
```python
class Rectangle:
    def __init__(self, width, height):
        self.width = width
        self.height = height

    def calculate(self):
        return self.width * self.height


r1 = Rectangle(3, 5)
r2 = Rectangle(7, 9)
print(r1.calculate(), r2.calculate())


```

## demo56_inherit

```python
class Emp:
    gradeLevel = 6

    def startWork(self):
        pass

    def endWork(self):
        pass

    pass


class PM(Emp):
    def startWork(self):
        print('pm with grade=',self.gradeLevel,"start to work")
    pass


class RD(Emp):
    pass

pm1 = PM()
pm1.startWork()
rd1 = RD()
rd1.startWork()
print(Emp.gradeLevel, RD.gradeLevel, PM.gradeLevel)
RD.gradeLevel = 7
print(Emp.gradeLevel, RD.gradeLevel, PM.gradeLevel)
Emp.gradeLevel = 8
print(Emp.gradeLevel, RD.gradeLevel, PM.gradeLevel)
del RD.gradeLevel
print(Emp.gradeLevel, RD.gradeLevel, PM.gradeLevel)

```

## demo57_instance
```
sum = 0

l1 = [10, "300", 35, None, 24, "---", 29, "Hello", 31, 5]

for l in l1:
    if isinstance(l, int):
        sum += l
print(f'sum={sum}')

```

## demo58_series1

```python
import pandas as pd
import numpy as np

l1 = [3, 1, 4, 5, 9, -2, 8]
a1 = np.array(l1)
pd1 = pd.Series(l1)
pd2 = pd.Series(a1)
print(type(l1), type(a1), type(pd1), type(pd2))
print(pd1 == pd2)
# check pd1 attributes
print(type(pd1.values), pd1.values)
print(type(pd1.index), pd1.index)

pd3 = pd.Series([4, 7, -5, 3],
                index=['Nangang', 'Taipei', 'Banqiao', 'Taoyuan'])
print(pd3)
print(pd3.values)
print(type(pd3.index), pd3.index)
# access
print('access 1 by 1==>', type(pd1[0]), pd1[0], pd1[2])
print('access multiple==>', type(pd1[[0, 2]]), pd1[[0, 2]],sep='\n')
print(pd3['Taipei'])
print(pd3[['Nangang','Banqiao']])


```

## demo59_series2
```Python
import pandas as pd

d1 = {'poop': 35, 'bdpy': 35, 'andbiz2': 28, 'testit': 14}
s1 = pd.Series(d1)
print(d1)
print(s1)
print('c#' in d1, 'C#' in s1)
print('poop' in d1, 'poop' in s1)
l1 = ['andbiz2', 'testit', 'poop', 'bdpy']
s2 = pd.Series(d1, index=l1)
print(s2)
# add index more than dictionary
l2 = ['arduino', 'andbiz2', 'testit', 'andbiz3', 'poop', 'bdpy', 'pykt']
s3 = pd.Series(d1, index=l2)
print(s3)
print('method1 use pd\n', pd.isna(s3))
print('method2 use pd\n', pd.isnull(s3))
print('method3 use series instance\n', s3.isna())
print('method4 use series instance\n', s3.isnull())

d2 = {'poop': "Mark", 'bdpy': None, 'andbiz2': None, 'testit': 'Frank'}
s4 = pd.Series(d2, index=l2)
print(s4)
print(s4.isna())
print(s4.isnull())

```

## demo60_series3
```
import pandas as pd

s1 = pd.Series([1000, 800, 500, 300],
               index=['nangang', 'taipei', 'banqiao', 'taoyuan'])
s2 = pd.Series([500, 300, 400],
               index=['hsinchu', 'taichung', 'tainan'])
s3 = pd.Series([900, 1200, 1500, 1300],
               index=['nangang', 'taipei', 'banqiao', 'taoyuan'])
s4 = pd.Series([1500, 350, 480],
               index=['hsinchu', 'taichung', 'tainan'])
print(s1)
print(s2)
print(s1 + s2)
print(s1 + s3)
print(s2 + s4)
# append
l1 = [1, 2, 3]
l1.append(4)
print(l1)
s5 = s1.append(s2)
s5.index.name = 'station'
s5.name = 'sold(k)'
print(s1)
print(s3)
print(s5)
# change index name (but keep values unchanged)!!
s5.index = ['taipei', 'taichung', 'tainan', 'banqiao', 'nangang',
            'hsinchu', 'taoyuan']
print(s5)
# use index to generate a new Series with index order
s6 = pd.Series(s1.append(s2),
               index=['taipei', 'taichung', 'tainan', 'banqiao', 'nangang',
                      'hsinchu', 'taoyuan'])
print(s6)




```

## demo61_series4
```python
import pandas as pd

dict1 = {'course': ['poop', 'bdpy', 'pytk', 'aiocv'],
         'year': [2018, 2017, 2019, 2020],
         'slide': [200, 250, 230, 300]}
dataFrame1 = pd.DataFrame(dict1)
print(type(dict1), type(dataFrame1))
print(dataFrame1.head(n=2))
dataFrame2 = pd.DataFrame(dict1,
                          columns=['course', 'slide', 'year', 'instructor'])
print(dataFrame2)
dataFrame3 = pd.DataFrame(dict1, index=['c1', 'c2', 'c3', 'c4'],
                          columns=['course', 'slide', 'year', 'instructor'])
print(dataFrame3)
print(dataFrame3.columns)
print(dataFrame3.index)
# slice data (choose columns)
print(type(dataFrame3), type(dataFrame3['course']),
      type(dataFrame3[['course', 'slide']]))
# slice data (choose rows)
print(type(dataFrame3.loc['c1']), dataFrame3.loc['c1'], sep='\n')
print(type(dataFrame3.loc[['c1', 'c3']]),
      dataFrame3.loc[['c1', 'c3']], sep='\n')
dataFrame3['year'] = 2020
print(dataFrame3)
dataFrame3['year'] = [2020, 2020, 2020, 2021]
print(dataFrame3)


```

## demo62_series5
```python
import pandas as pd

d1 = {'course': ['poop', 'bdpy', 'pykt', 'aiocv'],
      'year': [2018, 2017, 2019, 2020],
      'slide': [200, 250, 230, 300]}
pd1 = pd.DataFrame(d1)
print(pd1)
print("----")
s1 = pd.Series(['Taipei', 'Hsinchu', 'Taichung', 'Kaohsiung'],
               index=[0, 1, 2, 3])
pd1['location'] = s1
s2 = pd.Series(['remote', 'local'], index=[0, 3])
pd1['method'] = s2
pd1['heavy'] = pd1['slide'] >= 250
print(pd1)
del pd1['slide']
print(pd1)

```

## demo63_pandas1
```python
import pandas as pd

dict1 = {'poop': {2019: 5, 2020: 8},
         'bdpy': {2018: 5, 2019: 7, 2020: 10}}
df1 = pd.DataFrame(dict1)
print(df1)
df2 = df1.T
print(df2)

df3 = pd.DataFrame(dict1, index=[2018, 2019, 2020, 2021])
print(df3)
print(type(df3.values))
print(df3.values)
dict2 = {'poop': {2019: 5, 2020: 8},
         'bdpy': {2018: 5, 2019: 7, 2020: 'not yet'}}
df4 = pd.DataFrame(dict2, index=[2018, 2019, 2020, 2021])
print(df4)
print(type(df4.values), type(df4.values[0, 0]),
      type(df4.values[2, 1]))
print(df4.values[0, 0], df4.values[2, 1])
print(df4.values)
df4.iloc[0, 0] = -1
df4.iloc[2, 1] = 50
print(df4)

```

## demo64_index1

```python
import pandas as pd
import numpy as np

object1 = pd.Series(range(5), index=['p', 'q', 'r', 's', 't'])
print(object1)
index1 = pd.Index(['C', 'D', 'E', 'F', 'G', 'H'], dtype='object')
print(type(index1), index1)
print(index1[:3], index1[4:])
i1 = pd.Index(np.arange(3))
i2 = list(np.arange(3))
print(i1, i2)

data1 = ['Nangang', 'Taipei', 'Banqiao']
s1 = pd.Series(data1, index=i1)
print(s1)
s2 = pd.Series(data1, index=i2)
print(s2)
print(2 in i1, 2 in i2)
index3 = pd.Index(['Taipei', 'Taipei', 'Taipei'])
print(index3)
s3 = pd.Series(data1, index=index3)
print(s3)
print("result=\n", s3['Taipei'])


```

## demo65_index2
```python
import pandas as pd
import numpy as np

s1 = pd.Series([20, 15, 18, 37, 25],
               index=['mar', 'jan', 'feb', 'may', 'apr'])
s2 = s1.reindex(['jan', 'feb', 'mar', 'apr', 'may', 'jun'])
print(s2)
s3 = pd.Series(["L", "M", "S"], index=[0, 5, 10])
print(s3)
s4 = s3.reindex(range(15), method='ffill')
print(s4)
s5 = pd.DataFrame(np.arange(16).reshape(4, 4), index=[1, 2, 3, 4],
                  columns=['Kotlin', 'Swift', 'C++', 'Java'])
print(s5)
s6 = s5.reindex(columns=['objC', 'Kotlin', 'Swift', 'Java', 'C++'])
print(s6)

```

## demo66_drop
```python
import pandas as pd

dict1 = {'station': ['Nangang', 'Taipei', 'Banqiao', 'Taoyuan', 'Hsinchu'],
         'order': [1, 2, 3, 4, 5],
         'backOrder': [5, 4, 3, 2, 1]}
df1 = pd.DataFrame(dict1)
print(df1)
print(df1.drop(2))
print(df1.drop([1, 3]))
print(df1.drop('order', axis=1))
print(df1.drop(['order', 'backOrder'], axis=1))
print(df1.drop(['order', 'backOrder'], axis='columns'))
df1.drop([1, 4], inplace=True)
print(df1)


```

## demo67_functional
```python

import pandas as pd
import numpy as np

df1 = pd.DataFrame(np.random.randn(6, 7),
                   index=list(range(0, 12, 2)),
                   columns=list(range(0, 7, 1)))
print(df1)
func1 = lambda x: x.max() - x.min()
result1 = df1.apply(func1)
print(type(result1), result1)
result2 = df1.apply(func1, axis=1)
print(type(result2), result2)
result3 = df1.apply(func1, axis='columns')
print(type(result3), result3)

func2 = lambda x: pd.Series([x.min(), x.max()], index=['min', 'max'])
result4 = df1.apply(func2)
print(type(result4), result4, sep='\n')
result5 = df1.apply(func2, axis=1)
print(type(result5), result5, sep="\n")

print(df1)
df1.sort_index()
print(df1)
print("after sort")
df1.index = df1.iloc[:, 1]
print(df1.sort_index())
df1.columns = df1.iloc[2, :]
print("sort by axis=1")
df2 = df1.sort_index(axis=1)
print(df2)

```

## demo68_function2
```python
import pandas as pd
import numpy as np

df3 = pd.DataFrame(np.random.randn(6, 7),
                   index=list(range(0, 12, 2)),
                   columns=list(range(0, 7, 1)))
print(df3.sort_values(by=3))

s1 = pd.Series([4, -15, 7, 7, 2, 2, 0, 0, 4])
print(s1.rank())
print(s1.rank(method='first')) # same value, use order to assign rank
print(s1.rank(method='min'))
print(s1.rank(method='max'))

```

## demo69_function3
```python

import pandas as pd
import numpy as np

df1 = pd.DataFrame(np.random.randn(6, 7),
                   index=list(range(0, 12, 2)),
                   columns=list(range(0, 7, 1)))
print(df1)
print(df1.sum())
print(df1.sum(axis='columns'))
df1.iloc[2, 3] = np.NaN
print(df1.sum())
print(df1.sum(skipna=False))
print(df1.sum(axis='columns',skipna=False))
print(df1.idxmax())
print(df1.idxmin())
print(df1.idxmax(axis='columns'))
print(df1.idxmin(axis='columns'))
print(df1.cumsum())
print(df1.cummax())
print(df1.describe())
```

## demo70_functional4
```python
import pandas as pd

s1 = pd.Series([1, 1, 2, 3, 5, 7, 9])
print(s1.pct_change())
df1 = pd.DataFrame({'JP': [100, 200, 300, 400, 500],
                    'KR': [250, 200, 150, 100, 50]})
print(df1)
print(df1['JP'].corr(df1['KR']))
print(df1['JP'].cov(df1['KR']))
print(df1.corr())
s2 = pd.Series(['S', 'S', 'S', 'M', 'L', 'S', 'M', 'L', 'S', 'M', 'L'])
print(s2.value_counts())
print(pd.value_counts(s2))
df2 = pd.DataFrame({'JP': [1, 3, 4, 1, 4, 5],
                    'KR': [2, 3, 3, 2, 4, 2],
                    'TW': [5, 3, 1, 2, 4, 4]})
print(df2)
result = df2.apply(pd.value_counts)
print(result)
result2 = df2.apply(pd.value_counts).fillna(0)
print(result2)



```
## cd C:\Python36\Python36\Scripts

```
import pandas as pd
import numpy as np
df1 = pd.DataFrame(np.random.randn(6,7),index=list(range(0,12,2)),columns=list(range(0,7,1)))
df1
df1[:]
df1[2:]
df1[:2]
df1.iloc[:,:]
df1.iloc[:2,:]
df1.iloc[2:,:]
df1.iloc[:,:2]
df1.iloc[:,2:]
df1.iloc[1:3, 2:4]

```

## worldbank
* (worldbank)[https://data.worldbank.org/]
```python
import pandas as pd

data1 = pd.read_csv('data/data71.csv',
                    skiprows=4)
print(type(data1))
print(data1.columns)
print(data1.shape)
print(data1.head())
print(data1['Country Name'].values)
data2 = data1[data1['Country Name']=='Caribbean small states']
print(data2.shape)
```

## demo72_worldbank_demo
```python
import pandas as pd
from pandas_datareader import wb
import numpy as np

data = wb.download(
    indicator='SE.PRM.TENR',
    country=['all'],
    start=2002,
    end=2020
)
print(data.shape)
print(data.head())
data2 = data.reset_index()
print(data2.shape)
data3 = data2[data2.country == 'Caribbean small states']
print(data3.shape)
print(data3)
print(np.unique(data2.country, return_counts=True))
print(np.unique(data2.year, return_counts=True))
maxData = data2.groupby(['country'])['SE.PRM.TENR'].transform(max)
print(maxData.head())
index = maxData == data2['SE.PRM.TENR']
print(index)
print(np.unique(index, return_counts=True))
data4 = data2[index]
print(data4)
data5 = data4.sort_values(by='SE.PRM.TENR', ascending=False)
print(data5.head(n=20))
```

# Day4

## get world bank GDP Data
* (GDP Data)[https://data.worldbank.org/indicator/NY.GDP.PCAP.KD.ZG]
### demo73_worldbank_lab2
```python
import pandas as pd
import numpy as np
df1 = pd.read_csv("data/demo73.csv", skiprows=4)
print(df1.columns)
print(df1['Country Name'].values)
countries = ['Portugal', 'Italy', 'Ireland', 'Greece', 'Spain']
df2 = df1[df1['Country Name'].isin(countries)]
print(df2.head())
years = [str(x) for x in list(range(1960, 2020))]
print(years)
df3 = pd.melt(df2, 'Country Name',
              var_name='year', value_name='GDPPerCapitalGrowth',
              value_vars=years)
print(df3.head())
flattenGDPTable = pd.melt(df1, 'Country Name',
              var_name='year', value_name='GDPPerCapitalGrowth',
              value_vars=years)
print(flattenGDPTable.columns)
print(flattenGDPTable.shape)
MaxFlattenGDPTable \
    = flattenGDPTable.groupby(['Country Name'])['GDPPerCapitalGrowth'].transform(max)
print(MaxFlattenGDPTable.head())
index = MaxFlattenGDPTable == flattenGDPTable['GDPPerCapitalGrowth']
print(np.unique(index, return_counts=True))
maxGDPTable = flattenGDPTable[index]
print(maxGDPTable.head(n=20))
sortedMaxGDP = maxGDPTable.sort_values(by='GDPPerCapitalGrowth',
                                       ascending=False)
print(sortedMaxGDP.head(n=20))
```

## demo74
* (net enrollment rate)[https://data.worldbank.org/indicator/SE.PRM.TENR]
### demo74_corr_from_enrollment
```python
import pandas as pd

data1 = pd.read_csv('data/demo74.csv', skiprows=4)
print(data1.shape)
print(data1.columns)
years = [str(x) for x in list(range(1960, 2020))]
flattenData1 = pd.melt(data1, 'Country Name',
                       var_name='year', value_name='SE.PRM.TENR',
                       value_vars=years)
print(flattenData1.shape)
print(flattenData1.head())
franceData = flattenData1[flattenData1['Country Name'] == 'France']
franceData.index = franceData['year']
print(franceData)
germanData = flattenData1[flattenData1['Country Name'] == 'Germany']
germanData.index = germanData['year']
print(germanData)
df1 = pd.DataFrame({'fr': franceData['SE.PRM.TENR'], 'de': germanData['SE.PRM.TENR']})
print(df1)
print(df1['fr'].corr(df1['de']))
print(df1['fr'].cov(df1['de']))
print(df1.corr())
```
## demo75
* demo75_write_out
```
import sqlite3

import pandas as pd

data1 = pd.read_csv('data/demo74.csv', skiprows=4)
print(data1.shape)
print(data1.columns)
years = [str(x) for x in list(range(1960, 2020))]
flattenData1 = pd.melt(data1, 'Country Name',
                       var_name='year', value_name='SE.PRM.TENR',
                       value_vars=years)
flattenData1.to_csv('data/demo75.csv')
flattenData1.to_excel('data/demo75.xlsx', sheet_name='net_enrollment')
connection1 = sqlite3.connect('data/demo75.sqlite')
flattenData1.to_sql(name='net_enrollment',con=connection1)
connection1.close()


```
* pip show openpyxl

* https://sqlitebrowser.org/
## demo76
* demo76_cov19
* (github_cov19)[https://raw.githubusercontent.com/datasets/covid-19/master/data/countries-aggregated.csv]
```python
import pandas as pd
from pprint import pprint
import pycountry
import plotly.express as px

PATH = 'data/demo76_cov19.csv'
df1 = pd.read_csv(PATH)
print(df1.head())
print('before change')
print(df1.loc[df1['Country'] == 'Taiwan*'])
# change Taiwan* to Taiwan
df1.loc[df1['Country'] == 'Taiwan*', 'Country'] = "Taiwan"
print(df1.loc[df1['Country'] == 'Taiwan'])
list_countries = df1['Country'].unique().tolist()
pprint(list_countries)
d_country_code = {}
for country in list_countries:
    try:
        country_data = pycountry.countries.search_fuzzy(country)
        country_code = country_data[0].alpha_3
        d_country_code.update({country: country_code})
    except:
        print('could not add ISO 3 code for ==>', country)
        d_country_code.update({country: ' '})



```

## demo77
```
import numpy as np
from matplotlib import pyplot as plt

l1 = [1, 2, 3, 4, 5]
l2 = [5, 4, 3, 2, 1]
print(l1 + l2)
a1 = np.array(l1)
a2 = np.array(l2)
print(a1 + a2)
print(a1 - a2, a1 * a2, a1 / a2, a1 ** a2)
print(6 * a1 + 4 * a2 ** 2)
# 'o', '.','^','s'
# plt.plot(l1, 's')
# plt.plot(l1, l1, 's')
# plt.plot(l1, l2, 's')
plt.plot(a1, a2, 'rs-')
plt.plot(a1, a2 ** 2 - 2 ** a2 + 5, 'go-')
plt.xlabel('quantity')
plt.ylabel('probability')
plt.title("demo77 numpy to matplotlib")
plt.grid()
plt.show()



```

## demo78
* demo78_plot2
```
import numpy as np
from matplotlib import pyplot as plt

# a1 = np.array(range(10))
a1 = np.arange(0, 10, 0.25)
a2 = a1 ** 2
plt.plot(a1, a2, 'o-')
plt.show()

b1 = np.arange(0, 2, 0.1)
b2 = b1 ** 2
b3 = b1 ** 3
b4 = b1 ** 4
plt.plot(b1, b2, 'ro-')
plt.plot(b1, b3, 'g.--')
plt.plot(b1, b4, 'b^-')
plt.axis([-2, 2, -2, 2])
plt.show()


```
## keep python up2date
### under pyconsole(inside pycharm)
* inside SC
```python
import sys
!{sys.executable} -m pip install --upgrade pip  -i https://artifactory.global.standardchartered.com/artifactory/api/pypi/pypi/simple/
```
* outside SC使用自己電腦外網 
```python
import sys
!{sys.executable} -m pip install --upgrade pip  
```

## demo79
```
import numpy as np
from matplotlib import pyplot as plt

b1 = np.arange(0, 2, 0.1)
plt.plot(b1, b1, 'r--', b1, b1 ** 2, 'g.-', b1, b1 ** 3, 'b*--', linewidth=3)
plt.show()

```

## demo80_subplot
```
import matplotlib.pyplot as plt
import numpy as np

fig = plt.figure()
ax1 = fig.add_subplot(2, 2, 1)
ax2 = fig.add_subplot(2, 2, 2)
ax3 = fig.add_subplot(2, 2, 3)
ax4 = fig.add_subplot(2, 2, 4)
plt.plot(np.random.randn(20).cumsum(), 'r--')
ax1.hist(np.random.randn(200), bins=20, color='c', alpha=1)
ax2.scatter(np.arange(30), np.arange(30) + 3 * np.random.randn(30))
plt.show()

fig, axes = plt.subplots(2, 2)
print(type(axes), axes)
axes[0, 0].hist(np.random.randn(50), bins=20, color='m', alpha=1)
axes[1, 1].scatter(np.arange(30), np.arange(30) + 3 *
                   np.log(10) * np.random.randn(30))
plt.show()

```

## demo81_pandas_to_plot
```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

randomSequences = pd.DataFrame(np.random.normal(1.0, 0.07, (100, 8)))
print(randomSequences.head())
accumulates = randomSequences.cumprod()
accumulates.plot()
#randomSequences.plot()
plt.show()
```

## demo82_animation
```python
import matplotlib.animation as animation
import matplotlib.pyplot as plt
import numpy as np

fig, ax = plt.subplots()

x = np.arange(0, 2 * np.pi, 0.01)
line, = ax.plot(x, np.sin(x))


def animate(i):
    line.set_ydata(np.sin(x + i / 5.0))
    return line,


def init():
    line.set_ydata(np.ma.array(x, mask=True))
    return line,


anim = animation.FuncAnimation(fig, animate, np.arange(1, 400, 0.1),
                               init_func=init,
                               interval=50)
plt.show()


```

## demo83_pdf
```
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm

mu = 80
sigma = 8
x = mu + sigma * np.random.randn(10000)
print(len(x))
num_bins = 50
n, bins, patches = plt.hist(x, num_bins, density=1,
                            facecolor='blue', alpha=0.5)
y = norm.pdf(bins, mu, sigma)
plt.plot(bins, y, "r*-")
plt.show()


```

## demo84_population
* (population)[https://data.worldbank.org/indicator/SP.POP.TOTL]

```
import pandas
import matplotlib.pyplot as plt
from matplotlib import rc

data = pandas.read_csv('data/demo84.csv', skiprows=4)
print(data.head())
print(data.shape)
data.to_excel('data/output84.xlsx', sheet_name='pandas_generated')
print(data.describe())
data['created'] = data['1960'] + data['1980']
print(data.columns)
ausData = data[data['Country Code'] == 'AUS']
print(ausData.shape)
print(ausData['1980'])
selected_years = ['1960', '1970', '1980']
print(plt.style.available)
font1 = {'family': 'Source Code Pro'}
rc('font', **font1)
plt.style.use('fast')
ausData.plot(kind='bar', y=selected_years, fontsize=12)
plt.show()

```

* Practice
* (Taipei realtime)[https://taipeicity.github.io/traffic_realtime/]
* (json api)[https://tcgbusfs.blob.core.windows.net/blobyoubike/YouBikeTP.json]

## demo85_youbike

```
# 手動建一個目錄map

import folium
import json

taipei = [25.034132, 121.564153]
zoom = 15
map_osm = folium.Map(location=taipei, zoom_start=zoom)

file1 = open('data/demo85.json', encoding='utf8')
resultText = file1.read()
file1.close()
print(type(resultText), resultText[:50])
result = json.loads(resultText)
print(type(result))
data = result["retVal"]
# for k in data:
#     print(k)
for v in data.values():
    coord = [v['lat'], v['lng']]
    print(coord)
    folium.Marker(coord, icon=None, popup=None).add_to(map_osm)
map_osm.save('map/demo85.html')


```
## https://data.taipei/
* (勞基法)[https://data.taipei/#/dataset/detail?id=23630879-4926-4877-a48a-a0ae6cc2f7d5]

### lab86_read_labor_law
```
import pandas

data1 = pandas.read_csv('data/demo86.csv')
print(data1.head())
print(data1.columns)
print(data1.info())
data1Grouped = data1[['處分字號', '違反勞動基準法條款']] \
    .groupby(['違反勞動基準法條款']).count()
print(data1Grouped.head(10))
result = data1Grouped.sort_values('處分字號', ascending=False)
print(result.head(10))

data2Grouped = data1[['處分字號', '違反勞動基準法條款','違反法規內容']] \
    .groupby(['違反勞動基準法條款','違反法規內容']).count()
print(data2Grouped.head(10))
result2 = data2Grouped.sort_values('處分字號', ascending=False)
print(result2.head(10))
```

### https://data.taipei/#/dataset/detail?id=a835f3ba-7f50-4b0d-91a6-9df128632d1c

* demo87_recycle_bin
```
import folium
import pandas as pd

sample_data = pd.read_csv('data/shilin.csv', sep=',')
print(sample_data.shape)
print(sample_data.columns)
sample_data.columns = ['section', 'road', 'road_detail',
                       'lon', 'lat', 'extra']
print(sample_data.columns)
taipei = [25.095911, 121.518301]
zoom = 15
map_osm = folium.Map(location=taipei, zoom_start=zoom)

for i in range(len(sample_data)):
    coord = [sample_data.iloc[i, 4], sample_data.iloc[i, 3]]
    message = f"({i})[{sample_data.iloc[i,1]}]{sample_data.iloc[i,2]}"
    icon1 = folium.Icon(color='red', icon='info-sign')
    folium.Marker(coord, icon=icon1, popup=message).add_to(map_osm)

spot1 = [24.773071, 121.009916]
folium.CircleMarker(spot1, radius=500, popup='Science Park',
                    fill_color='#C0FFEE').add_to(map_osm)
spot2 = [24.777657, 121.043790]
folium.Circle(spot2, radius=500, popup='ITRI',
              fill_color='#FFC0EE').add_to(map_osm)
map_osm.save('map/demo87.html')


```

# spark installation
## download 2.4.6
## download winutil
* (origin)[https://github.com/steveloughran/winutils]
* (newer)[https://github.com/cdarlint/winutils]
## unzip spark2.4.6
## copy winutil hadoop 2.7 bin ==> spark \bin
## https://confluence.jetbrains.com/display/JBR/JetBrains+Runtime
```
scala> val textFile = sc.textFile("aa.txt")
textFile: org.apache.spark.rdd.RDD[String] = aa.txt MapPartitionsRDD[1] at textFile at <console>:24

scala> textFile.count()
res0: Long = 5
```



## demo87

```python
import sys
import pyspark
from pyspark import SparkContext
from pprint import pprint
pprint(sys.path)
print(pyspark.__version__)
sc = SparkContext("local","simple app")
print(sc)
```



## demo88_open_sqlite

```
import sqlite3

connection1 = sqlite3.connect('data/demo88.sqlite')
DROP_DDL = "DROP TABLE IF EXISTS EMPLOYEE"
CREATE_DDL = '''
CREATE TABLE EMPLOYEE
(ID INTEGER PRIMARY KEY,
NAME TEXT NOT NULL,
AGE INT NOT NULL,
DEPT INT,
ADDRESS CHAR(50));
'''
connection1.execute(DROP_DDL)
connection1.execute(CREATE_DDL)
connection1.close()

```
## demo89.py
```
import sqlite3
import time

connection1 = sqlite3.connect('data/demo88.sqlite')
emp1 = {'NAME': "Mark", 'AGE': 43, 'DEPT': 1, "ADDR": 'Taipei'}
emp2 = {'NAME': "John", 'AGE': 42, 'DEPT': 2, "ADDR": 'Hsinchu'}
emp3 = {'NAME': "Ken", 'AGE': 44, 'DEPT': 1, "ADDR": 'Taipei'}
emp4 = {'NAME': "Tim", 'AGE': 45, 'DEPT': 2, "ADDR": 'Kaohsiung'}

employees = [emp1, emp2, emp3, emp4]
INSERT_DML = '''INSERT INTO EMPLOYEE(NAME, AGE, DEPT, ADDRESS)
 VALUES(?,?,?,?)'''
start_time = time.time()
for i in range(1000):
    for e in employees:
        connection1.execute(INSERT_DML, (e['NAME'], e['AGE'], e['DEPT'],
                                         e['ADDR']))

connection1.commit()
end_time = time.time()
connection1.close()
print(f"total spend {end_time-start_time} seconds")



```

#### data prepare
```python
import tensorflow as tf
from keras import datasets
print(tf.__version__)
datasets.mnist.load_data()
datasets.boston_housing.load_data()
datasets.imdb.load_data()
datasets.reuters.load_data()
datasets.imdb.get_word_index()
datasets.reuters.get_word_index()
```

# new 4 days
## [MLDL](https://hackmd.io/@O0oTwRN6TESqcS_-MQZDVQ/BJPVjRS7w)
