Python
random.sample != random.choices
Interesting python projects
- https://github.com/trending/python
- https://github.com/vinta/awesome-python
- https://github.com/gleitz/howdoi
- https://github.com/mooz/percol
- https://github.com/blaze/blaze
- https://github.com/ironmussa/Optimus
- https://github.com/altair-viz/altair
- https://github.com/fabric/fabric
- https://you-get.org
- https://github.com/jaraco/path.py
- https://github.com/gorakhargosh/watchdog
- https://github.com/dbader/schedule
- https://github.com/mahmoud/boltons
- https://github.com/vinta/awesome-python#natural-language-processing
- https://github.com/gaojiuli/toapi
- https://github.com/deanmalmgren/textract
- http://html.python-requests.org
- https://github.com/buriy/python-readability
- https://github.com/codelucas/newspaper support keywords
- https://github.com/michaelhelmick/lassie
- https://github.com/Alir3z4/html2text
- https://github.com/trending/python Encode trending as part of your news flow
Unit test
Mock
import datetime
from unittest.mock import Mock, patch
datetime_mock = Mock(wraps=datetime.datetime)
datetime_mock.now.return_value = datetime.datetime(1999, 1, 1)
with patch('datetime.datetime', new=datetime_mock):
pass # do something that uses datetime.datetime
Read/write format
import yaml # pip install PyYAML
with open("path/to/file", "r") as fh:
value = yaml.safe_load(fh)["key"]
SendGrid
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail
def send_text(subject: str, content: str):
api_key = CONFIG["sendgrid_api_key"]
message = Mail(
from_email=CONFIG["sender_email"],
to_emails=CONFIG["receiver_email"],
subject=subject,
html_content=content)
try:
sg = SendGridAPIClient(api_key)
response = sg.send(message)
assert response.status_code == 200, response.body
except Exception as e:
print(e)
Misc
Parse URL:
from urllib.parse import urlparse
urlparse(url).netloc # e.g. www.google.com
Selector:
from parsel import Selector
import requests
response = requests.get(url)
response.raise_for_status()
selector = Selector(text=response.text)
for div in selector.xpath("//div[contains(@class, 'class-name')]"):
html_content = div.extract()
element_id = div.attrib["id"]
for code_text in div.xpath("//code/text()"):
code = code_text.extract()
html to text
import html2text
text_maker = html2text.HTML2Text()
text_maker.ignore_emphasis = True
text = text_maker.handle(html)
Datatime
import datetime
def get_now_iso_str():
return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()
def epoch_iso_str(epoch):
return datetime.datetime.utcfromtimestamp(epoch).replace(tzinfo=datetime.timezone.utc).isoformat()
Google Calendar API
https://developers.google.com/calendar/quickstart/python
"""
https://developers.google.com/resources/api-libraries/documentation/calendar/v3/python/latest/calendar_v3.events.html
"""
from __future__ import print_function
import pytz
from tzlocal import get_localzone # $ pip install tzlocal
import datetime
import pickle
import os.path
import dateutil.parser
import sys
import os
from termcolor import colored
try:
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
except ModuleNotFoundError:
print("pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib")
# If modifying these scopes, delete the file gcal_token.pickle.
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
def parse_event_time(event_time):
parsed = event_time.get('dateTime', None)
if parsed is None:
return None
return dateutil.parser.parse(parsed).astimezone(get_localzone())
def get_prev_week_events():
creds = None
# The file gcal_token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('gcal_token.pickle'):
with open('gcal_token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
os.getenv("HOME") + '/.google/credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('gcal_token.pickle', 'wb') as token:
pickle.dump(creds, token)
service = build('calendar', 'v3', credentials=creds)
calendars = { "example@gmail.com": "Work" }
ret = []
now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time
a_week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7)
a_week_ago = a_week_ago.astimezone(pytz.utc).isoformat()
for cid, name in calendars.items():
events_result = service.events().list(calendarId=cid, timeMin=a_week_ago, maxResults=100,
timeMax=now, singleEvents=True, orderBy='startTime').execute()
events = events_result.get('items', [])
for event in events:
start = parse_event_time(event["start"])
end = parse_event_time(event["end"])
if start and end:
summary = ""
if 'summary' in event:
summary = event['summary']
ret.append({
"start": start,
"end": end,
"event": "[%s] %s" % (name, summary)
})
return ret
Github API
from github import Github # pip install PyGithub
import os
import datetime
from tzlocal import get_localzone # $ pip install tzlocal
print('Loading Github data...')
github_events = []
github_histy_token = open(os.getenv("HOME") + "/.github_histy_token", "r").read().strip()
g = Github(github_histy_token)
user = g.get_user()
# https://developer.github.com/v3/repos/
# Personal Access Token: repo
for repo in user.get_repos(sort="updated"):
now = datetime.datetime.now()
if (now - repo.updated_at).days > 7:
break
commits = []
a_week_ago = now - datetime.timedelta(days=7)
for commit in repo.get_commits(since=a_week_ago, author=user):
git_commit = commit.commit
if git_commit.committer.name == "Linus Torvalds":
commits.append({
"date": git_commit.committer.date,
"msg": git_commit.message
})
if commits:
for commit in commits:
local_date = commit["date"].replace(tzinfo=datetime.timezone.utc).astimezone(get_localzone())
msg = commit["msg"].split("\n")[0]
github_events.append({
"event": f"[{repo.full_name}] {msg}",
"start": local_date
})
Twitter API
import twitter # pip install python-twitter
api = twitter.Api(consumer_key=getenv("TW_CONSUMER_KEY"),
consumer_secret=getenv("TW_CONSUMER_SECRET"),
access_token_key=getenv("TW_ACCESS_TOKEN"),
tweet_mode='extended',
access_token_secret=getenv("TW_ACCESS_TOKEN_SECRET"))
tweets = api.GetUserTimeline(screen_name="jack", count=500)
tweets = [ t.AsDict() for t in tweets ]