python

首页python
29
Dec
0

impala安装使用

pip install six
pip install bit_array
pip install thriftpy
pip install sasl-0.2.1-cp36-cp36m-win32.whl
pip install thrift_sasl
pip install bitarray-0.8.3-cp36-cp36m-win32.whl
pip install impyla
pip install pandas

测试代码
from impala.dbapi import connect
from impala.util import as_pandas
conn = connect(host='10.86.45.40', port=21050)
cur = conn.cursor()

cur.execute('SHOW TABLES')

data = as_pandas(cur)

print(data)

cur.execute('SELECT * FROM geelylogdatawarehouse limit 10')

cur.execute('SELECT row_number() over (order by appid) as rnum, * FROM geelylogdatawarehouse where ymd = 20171230 order by appid limit 10 offset 1000000')

cur.execute('SELECT count(*) FROM geelylogdatawarehouse where ymd = 20171230')

data = cur.fetchall()`

print(data)

description = cur.description

print(description)

for row in cur:

print(row[4])

data = as_pandas(cur)
print(data)

报错:
thriftpy.parser.exc.ThriftParserError: ThriftPy does not support generating module with path in protocol 'c'
解决:
根据提示找到parser.py文件 488行 if url_scheme == '': 修改为 if len(url_scheme) <= 1:

    1. whl文件需要到https://www.lfd.uci.edu/~gohlke/pythonlibs/下载
      或者使用附件里的版本
    2. centos下:sudo yum install gcc-c++ python-devel.x86_64 cyrus-sasl-devel.x86_64 然后pip install sasl
    3. 如果直接安装impyla则会报Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual
      C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools

whl.rar

26
Nov
0

CSRF Failed: CSRF token missing or incorrect

开启CSRF及
SESSION_COOKIE_HTTPONLY = True
CSRF_COOKIE_HTTPONLY = True
后,遇到错误:CSRF Failed: CSRF token missing or incorrect

解决方法一:

REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.TokenAuthentication',)
}

解决方法二:

MIDDLEWARE_CLASSES = (
'store.disable.DisableCSRF',
)

disable.py

class DisableCSRF(object):

def process_request(self, request):
    setattr(request, '_dont_enforce_csrf_checks', True)
22
Nov
0

python后台程序实现

方法1 使用nohup命令
nohup python work.py >my.log &

方法2 python-daemon
安装python-daemon

pip install python-daemon

编写入口程序
use_daemon.py

-- coding:utf-8 --

import daemon
from work import work
with daemon.DaemonContext():
work()

19
Nov
0

python模板注释

每次设置文件头麻烦 下次重装系统直接来复制

file -> settings -> editor -> file and code templetes -> python script

!/usr/bin/python3.6.7

-- coding: utf-8 --

@Time    : ${DATE} ${TIME}

@Author  : ${USER}

@Desc : ==============================================

Life is Short I Use Python!!!                      ===

If this runs wrong,don't ask me,I don't know why;  ===

If this runs right,thank god,and I don't know why. ===

Maybe the answer,my friend,is blowing in the wind. ===

======================================================

@Project : ${PROJECT_NAME}

@FileName: ${NAME}.py

@Software: ${PRODUCT_NAME}${PRODUCT_VERSION}

19
Oct
0

信号量Semaphore和互斥锁Mutex和condition对象和event对象

信号量

from multiprocessing import Process,Semaphore
import time,random

def go_wc(sem,user):
    sem.acquire()
    print('%s 占到一个茅坑' %user)
    time.sleep(random.randint(0,3)) #模拟每个人拉屎速度不一样,0代表有的人蹲下就起来了
    sem.release()

if __name__ == '__main__':
    sem=Semaphore(3)
    p_l=[]
    for i in range(10):
        p=Process(target=go_wc,args=(sem,'user%s' %i,))
        p.start()
        p_l.append(p)

    for i in p_l:
        i.join()
    print('============》')

互斥锁

from multiprocessing import Process, Lock
from time import sleep

def tack():
    search()
    lock.acquire()    # 上锁
    sleep(0.01)
    lock.release()    # 释放
 
 
if __name__ == '__main__':
    lock = Lock()    # 实例化一把锁
 
    for i in range(100):    # 模拟并发100个客户端抢票
        p =Process(target=tack)
        p.start()

condition对象

所谓条件变量,即这种机制是在满足了特定的条件后,线程才可以访问相关的数据。
它使用Condition类来完成,由于它也可以像锁机制那样用,所以它也有acquire方法和release方法,而且它还有wait,notify,notifyAll方法。

一个简单的生产消费者模型,通过条件变量的控制产品数量的增减,调用一次生产者产品就是+1,调用一次消费者产品就会-1.
使用 Condition 类来完成,由于它也可以像锁机制那样用,所以它也有 acquire 方法和 release 方法,而且它还有
wait, notify, notifyAll 方法。

import threading
import queue, time, random
 
# 产品类
class Goods:
    def __init__(self):
        self.count = 0
 
    def add(self, num=1):
        self.count += num
 
    def sub(self):
        if self.count >= 0:
            self.count -= 1
 
    def empty(self):
        return self.count <= 0
 
# 生产者
class Producer(threading.Thread):
 
    def __init__(self, condition, goods, sleeptime=1):
        threading.Thread.__init__(self)
        self.cond = condition
        self.goods = goods
        self.sleeptime = sleeptime
 
    def run(self):
        cond = self.cond
        goods = self.goods
        while True:
            # 锁住资源
            cond.acquire()
            goods.add()
            print("产品数量:", goods.count, "生产者线程")
            # 唤醒所有等待的线程 -> 其实就是唤醒消费者进程
            cond.notifyAll()
            # 解锁资源
            cond.release()
            time.sleep(self.sleeptime)
 
# 消费者
class Consumer(threading.Thread):
    def __init__(self, condition, goods, sleeptime=2):
        threading.Thread.__init__(self)
        self.cond = condition
        self.goods = goods
        self.sleeptime = sleeptime
 
    def run(self):
        cond = self.cond
        goods = self.goods
        while True:
            time.sleep(self.sleeptime)
            # 锁住资源
            cond.acquire()
            # 如无产品则让线程等待
            while goods.empty():
                cond.wait()
            goods.sub()
            print("产品数量:", goods.count, "消费者线程")
            # 解锁资源<br>            
            cond.release()
g = Goods() 
c = threading.Condition() 
pro = Producer(c, g) 
pro.start() 
con = Consumer(c, g) 
con.start()

event 对象

event 对象最好单次使用,就是说,你创建一个 event 对象,让某个线程等待这个对象,
一旦这个对象被设置为真,你就应该丢弃它。尽管可以通过 clear() 方法来重置 event 对
象,但是很难确保安全地清理 event 对象并对它重新赋值。很可能会发生错过事件、死锁
或者其他问题(特别是,你无法保证重置 event 对象的代码会在线程再次等待这个 event对象之前执行)。如果一个线程需要不停地重复使用 event 对象,你最好使用 Condition
对象来代替。下面的代码使用 Condition 对象实现了一个周期定时器,每当定时器超时的
时候,其他线程都可以监测到

import threading
import time
 
class PeriodicTimer:
    def __init__(self, interval):
        self._interval = interval
        self._flag = 0
        self._cv = threading.Condition()
 
    def start(self):
        t = threading.Thread(target=self.run)
        t.daemon = False
        t.start()
 
    def run(self):
        # Run the timer and notify waiting threads after each interval
        while True:
            time.sleep(self._interval)
            with self._cv:
                self._flag ^= 1
                self._cv.notify_all()
 
    def wait_for_tick(self):
        # wait for the next tick of the timer
 
        with self._cv:
            last_flag = self._flag
            while last_flag == self._flag:
                self._cv.wait()
 
ptimer = PeriodicTimer(2)
ptimer.start()
 
def countdown(nticks):
    while nticks > 0:
        ptimer.wait_for_tick()
        print('T-minus', nticks)
        nticks -= 1
 
def countup(last):
    n = 0
    while n < last:
        ptimer.wait_for_tick()
        print('Counting', n)
        n += 1
 
 
threading.Thread(target=countdown, args=(10,)).start()
threading.Thread(target=countup, args=(10,)).start()