https://stackoverflow.com/questions/25407550/how-do-i-log-from-my-python-spark-script

log_util.py

#!/usr/bin/python
# coding=utf-8

import socket


def main():
    print("hello ....")
    tags = dict()
    tags["flight"] = "MU2916"
    tags["key"] = "0001"
    # log_info(logger=None, appname="wordcount", map=map, message="test message")
    get_text( appname="wordcount", tags=tags, message="test message")
    get_text( appname="wordcount", message="test message")


# 获取本机IP
def get_host_ip():
    """
    查询本机ip地址
    :return:
    """

    try:
        s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 80))
        ip=s.getsockname()[0]
    finally:
        s.close()

    return ip


def get_text(appname='', tags=dict(),message=''):

    result ="@@"+ get_host_ip()+"@@"+appname+"@@"+str(tags)+"@@"+message+"@@"
    print result
    return result

if __name__ == "__main__":

    main()

main.py

    sc, sqlContext = s.startContext(appName)
    log4jLogger = sc._jvm.org.apache.log4j
    log4jLogger.LogManager.getLogger("com.umetrip.redis").setLevel(log4jLogger.Level.ERROR )
    log4jLogger.LogManager.getLogger("org.apache.spark").setLevel(log4jLogger.Level.WARN )
    log4jLogger.LogManager.getLogger("akka").setLevel(log4jLogger.Level.ERROR )
    global log
    log = log4jLogger.LogManager.getLogger("umetrip_pyspark")
    info=log_util.get_text( appname=appName, message="test message")
    log.warn(info)