利用Python从Steam导出用户游戏列表

奶牛关的用户游戏列表需要手动添加，这对于凯撒猫来说简直是痛苦万分，所以今天做了一个Python爬虫来获取自己的游戏列表（游戏名、游戏时间，etc）数据。本来计划是把这些数据转移到奶牛关，但是发现此时已经 10pm 了！凯撒猫还需要时间来做其他爱做的事情。

下面就是部分代码，请在GPLv3协议下使用这段代码。

##########################################################
# steam_games.py
# A spider that interprets game list of specific user.
#
# Author    : KaiserKatze
# License   : GPLv3
##########################################################
import re
import json
import requests
from bs4 import BeautifulSoup
from requests.compat import urljoin
#import steamapi

#API_KEY = None
encoding = "utf-8"
parser = "html.parser"

#steamapi.core.APIConnection(API_KEY)
request_headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip",
    "Connection": "keep-alive"
}

#def loadUser(id, name):
#    me = steamapi.user.SteamUser(userid=id, userurl=name)
#    return me

def loadGames(urlSteamGames):
    res = requests.get(urlSteamGames, headers=request_headers)
    if res.status_code == 200:
        html = res.text.encode(res.encoding).decode(encoding)
        return parseGamesPage(html)
       
def parseGamesPage(html):
    soup = BeautifulSoup(html, parser)
    pattern = "http://steamcommunity-a.akamaihd.net/public/javascript/profile_gameslist_functions.js\?v=\w+&l=\w+"
    nodes = soup.body("script", attrs={"language": "javascript"})
    if len(nodes) != 1:
        print "Error: Data script node should only appear once!"
    node = nodes[0]
    text = node.string.strip()
    lines = text.split("\n")
    line = lines[0].strip()
    line = line[14:-1]
    applist = json.loads(line)
    res = {}
    for app in applist:
        #app_id          = app["appid"]
        app_name        = app["name"].encode(encoding)
        try:
            app_time    = app["hours_forever"].encode(encoding)
        except:
            app_time    = "0.0"
        res[app_name] = app_time
    return res

def searchGame(app_name):
    urlCowlevelGame = "https://cowlevel.net/game/"
    urljoin(urlCowlevelGame, app_name)
def run():
    from optparse import OptionParser
    steamURLpattern = "^(?:http(?:s)?://)?steamcommunity.com/(?:(?:id/(?P<name>\S+?))|(?:profiles/(?P<id>\d+)))(?:/)?$"
    cowlevelURLpattern = "(?:http(?:s)?://)?cowlevel.net/people/(\w+)"
    parser = OptionParser()
    parser.add_option("--steam", type="string", dest="steam", help="Steam Profile URL")
    parser.add_option("--cowlevel", type="string", dest="cowlevel", help="Cowlevel URL")
    (options, args) = parser.parse_args()
    steamURL = options.steam
    #print "Steam URL\t:", steamURL
    if not steamURL:
        parser.error("Steam URL must be set!")
    m1 = re.match(steamURLpattern, steamURL)
    if not m1:
        parser.error("Steam URL is invalid!")
    urlCowlevel = options.cowlevel
    if not urlCowlevel:
        parser.error("Cowlevel URL must be set!")
    m2 = re.match(cowlevelURLpattern, urlCowlevel)
    if not m2:
        parser.error("Cowlevel URL is invalid!")
   
    #id = m1.group("id")
    #name = m1.group("name")
    #me = loadUser(id, name)
    #if not name:
    #    name = me.name
    #if not id:
    #    id = me.id
    #print "Player Name\t:", name
    #print "Player ID\t:", id
   
    username = m2.group(1)
    if not username:
        parser.error("Fail to interpret username from cowlevel URL.")
    urlSteamGames = urljoin(steamURL, "games/?tab=all&sort=name")
    #print "Steam Games URL\t:", urlSteamGames
    #print "Cowlevel URL\t:", urlCowlevel
    games = loadGames(urlSteamGames)
    for game in games:
        print games[game], "hours\t", game

if __name__ == "__main__":
    run()