超群.com的博客

<?php
// 输出Excel文件头，可把user.csv换成你要的文件名
header('Content-Type: application/vnd.ms-excel');
header('Content-Disposition: attachment;filename="user.csv"');
header('Cache-Control: max-age=0');
 
// 从数据库中获取数据，为了节省内存，不要把数据一次性读到内存，从句柄中一行一行读即可
$sql = 'select * from tbl where ……';
$stmt = $db->query($sql);
 
// 打开PHP文件句柄，php://output 表示直接输出到浏览器
$fp = fopen('php://output', 'a');
 
// 输出Excel列名信息
$head = array('姓名', '性别', '年龄', 'Email', '电话', '……');
foreach ($head as $i => $v) {
    // CSV的Excel支持GBK编码，一定要转换，否则乱码
    $head[$i] = iconv('utf-8', 'gbk', $v);
}
 
// 将数据通过fputcsv写到文件句柄
fputcsv($fp, $head);
 
// 计数器
$cnt = 0;
// 每隔$limit行，刷新一下输出buffer，不要太大，也不要太小
$limit = 100000;
 
// 逐行取出数据，不浪费内存
while ($row = $stmt->fetch(Zend_Db::FETCH_NUM)) {
 
    $cnt ++;
    if ($limit == $cnt) { //刷新一下输出buffer，防止由于数据过多造成问题
        ob_flush();
        flush();
        $cnt = 0;
    }
 
    foreach ($row as $i => $v) {
        $row[$i] = iconv('utf-8', 'gbk', $v);
    }
    fputcsv($fp, $row);
}

<?php
$s3Config = array(
    'foo' => array(
        'server' => '127.0.0.1',
        'database' => 'test',
        'user' => 'test',
        'password' => 'foobar',
        'domain' => 'http://s3.foobar.com'
    ),
 
    'bar' => array(
        'server' => '127.0.0.1',
        'database' => 'test',
        'user' => 'test',
        'password' => 'foobar',
        'domain' => 'http://s3.foobar.com'
    ),
);

<?php
/**
 * 统一文件存储
 *
 */
class Api_S3
{
    protected $_node;
 
    protected $_dir;
 
    protected $_config;
 
    /**
     * 构造函数
     *
     * @param string $node
     * @param string $dir
     * @param array $config
     */
    public function __construct($node, $dir = null, $config = null)
    {
        $this->_config = $config;
 
        $this->path($node, $dir, false);
    }
 
    /**
     * 设置文件路径
     *
     * @param string $node
     * @param string $dir
     * @return Api_S3
     */
    public function path($node, $dir, $connect = true)
    {
        $this->_node = $node;
        $this->_dir = empty($dir) ? 'fs' : $dir;
 
        if (empty($this->_config[$this->_node])) {
            throw new Cola_Exception('Api_S3: invalidate node');
        }
 
        if ($connect) {
            $this->_gridFS = $this->_gridFS();
        }
 
        return $this;
    }
 
    /**
     * GridFS
     *
     * @return MongDbGridFS
     */
    protected function _gridFS()
    {
        $mongo = new Cola_Com_Mongo($this->_config[$this->_node]);
 
        return $mongo->gridFS($this->_dir);
    }
 
    /**
     * 获得文件句柄
     *
     * @param string $name
     * @return MongoGridFSFile
     */
    public function file($name)
    {
        if (empty($this->_gridFS)) {
            $this->_gridFS = $this->_gridFS();
        }
 
        return $this->_gridFS->findOne(array('filename' => $name));
    }
 
    /**
     * 获得文件内容
     *
     * @param string $name
     */
    public function read($name)
    {
        $file = $this->file($name);
 
        return $file->getBytes();
    }
 
    /**
     * 写入文件
     *
     * @param string $name
     * @param string $data
     * @param array $extra
     * @param boolean $overWrite
     * @return boolean
     */
    public function write($name, $data, $extra = array(), $overWrite = false)
    {
        $extra = (array)$extra + array('filename' => basename($name));
 
        if ($filetype = $this->_type($name)) {
            $extra['filetype'] = $filetype;
        }
 
        if ($this->file($extra['filename'])) {
            if ($overWrite) {
                $this->delete($extra['filename']);
            } else {
                throw new Cola_Exception('Api_S3: file exists');
            }
        }
 
        return $this->_gridFS->storeBytes($data, $extra);
    }
 
    /**
     * 复制系统文件
     *
     * @param string $file
     * @param array $extra
     * @param boolean $overWrite
     * @return boolean
     */
    public function copy($file, $extra = array(), $overWrite = false)
    {
        $extra = (array)$extra + array('filename' => basename($file));
 
        if ($filetype = $this->_type($file)) {
            $extra['filetype'] = $filetype;
        }
 
        if ($this->file($extra['filename'])) {
            if ($overWrite) {
                $this->delete($extra['filename']);
            } else {
                throw new Cola_Exception('Api_S3: file exists');
            }
        }
 
        return $this->_gridFS->storeFile($file, $extra);
    }
 
    /**
     * 删除文件
     *
     * @param string $name
     * @return boolean
     */
    public function delete($name)
    {
        if (empty($this->_gridFS)) {
            $this->_gridFS = $this->_gridFS();
        }
 
        return  $this->_gridFS->remove(array('filename' => $name));
    }
 
    /**
     * 获得文件地址
     *
     * @param string $name
     * @param string $default
     * @return string
     */
    public function getUrl($name, $default = false)
    {
        $data = array(
            'domain' => rtrim($this->_config[$this->_node]['domain'], '/'),
            'path'   => $this->_node . (('fs' == $this->_dir) ? '' : $this->_dir),
            'name'   => $name
        );
        return  implode('/', $data);
    }
 
    /**
     * 设置文件属性
     *
     * @param string $name
     * @param array $attr
     * @return boolean
     */
    public function setAttr($name, $attr)
    {
        if (!$file = $this->file($name)) {
            throw new Cola_Exception('Api_S3: file not exists');
        }
 
        $file->file = $attr + $file->file;
 
        return $this->_gridFS->save($file->file);
    }
 
    /**
     * 获得文件属性
     *
     * @param string $name
     * @return array
     */
    public function getAttr($name)
    {
        $file = $this->file($name);
        return $file->file;
    }
 
    /**
     * 获得文件类型
     *
     * @param string $file
     * @return string
     */
    protected function _type($file)
    {
        return mime_content_type($file);
    }
}

<?php
$s3 = new Api_S3($node, $dir, $s3Config);
$s3->copy($file, array('filename' => $name, 'filetype' => $type));

<?php
$s3 = new Api_S3($node, $dir, $s3Config);
$file = $s3->file($name);
 
Cola_Response::lastModified($file->file['uploadDate']->sec);
Cola_Response::etag($file->file['md5']);
 
if (isset($file->file['filetype'])) {
    header("Content-Type: {$file->file['filetype']}");
}
 
echo $file->getBytes();

'''
Sparse Matrix
'''
import struct
import numpy as np
import bsddb
from cStringIO import StringIO
 
class DictMatrix():
    def __init__(self, container = {}, dft = 0.0):
        self._data  = container
        self._dft   = dft
        self._nums  = 0
 
    def __setitem__(self, index, value):
        try:
            i, j = index
        except:
            raise IndexError('invalid index')
 
        ik = ('i%d' % i)
        # 为了节省内存，我们把j, value打包成字二进制字符串
        ib = struct.pack('if', j, value)
        jk = ('j%d' % j)
        jb = struct.pack('if', i, value)
 
        try:
            self._data[ik] += ib
        except:
            self._data[ik] = ib
        try:
            self._data[jk] += jb
        except:
            self._data[jk] = jb
        self._nums += 1
 
    def __getitem__(self, index):
        try:
            i, j = index
        except:
            raise IndexError('invalid index')
 
        if (isinstance(i, int)):
            ik = ('i%d' % i)
            if not self._data.has_key(ik): return self._dft
            ret = dict(np.fromstring(self._data[ik], dtype = 'i4,f4'))
            if (isinstance(j, int)): return ret.get(j, self._dft)
 
        if (isinstance(j, int)):
            jk = ('j%d' % j)
            if not self._data.has_key(jk): return self._dft
            ret = dict(np.fromstring(self._data[jk], dtype = 'i4,f4'))
 
        return ret
 
    def __len__(self):
        return self._nums
 
    def __iter__(self):
        pass
 
    '''
    从文件中生成matrix
    考虑到dbm读写的性能不如内存，我们做了一些缓存，每1000W次批量写入一次
    考虑到字符串拼接性能不太好，我们直接用StringIO来做拼接
    '''
    def from_file(self, fp, sep = '\t'):
        cnt = 0
        cache = {}
        for l in fp:
            if 10000000 == cnt:
                self._flush(cache)
                cnt = 0
                cache = {}
            i, j, v = [float(i) for i in l.split(sep)]
 
            ik = ('i%d' % i)
            ib = struct.pack('if', j, v)
            jk = ('j%d' % j)
            jb = struct.pack('if', i, v)
 
            try:
                cache[ik].write(ib)
            except:
                cache[ik] = StringIO()
                cache[ik].write(ib)
 
            try:
                cache[jk].write(jb)
            except:
                cache[jk] = StringIO()
                cache[jk].write(jb)
 
            cnt += 1
            self._nums += 1
 
        self._flush(cache)
        return self._nums
 
    def _flush(self, cache):
        for k,v in cache.items():
            v.seek(0)
            s = v.read()
            try:
                self._data[k] += s
            except:
                self._data[k] = s
 
if __name__ == '__main__':
    db = bsddb.btopen(None, cachesize = 268435456)
    data = DictMatrix(db)
    data.from_file(open('/path/to/log.txt', 'r'), ',')

import timeit
timeit.Timer('foo = __main__.data[9527, ...]', 'import __main__').timeit(number = 1000)