百度云API(baiduPCS)实现服务器自动备份

百度云容量大,速度也不错,并且有个人云存储(PCS) API,不超量的情况下免费(量足够大),完全可以用程序实现文件管理,加入cron完全无人值守,本文是一个完整的服务器备份实例,修改了百度云的PHP API有bug,并且扩展了一些功能,尤其是API的上传,默认是把文件全部读入内存然后上传,如果文件非常大肯定不行的,改为边读文件边上传,每个文件上传都有进度显示,根据传输字节和文件大小计算有一定误差。

首先需要有个pcs的工程,http://developer.baidu.com/ms/pcs,进入控制台然后创建工程即可,可以得到工程的ID,API Key,Secret Key等,还可以进行其他一些设置。有了这些Key需要一个access token就可以和百度云通信了。代码位置比较分散就不打包了(关键还要修改代码中的一些敏感信息比较麻烦),类名和文件名一致,鼠标指向代码处有复制按钮,只有核心逻辑,所以代码还需要自己加工。

本文用的BaiduPCS API(有极个别bug修正),BaiduPcs

需要在安全设置里面设置回调地址和绑定的域名,这个在API手册上有详细说明。2014-10-12_121104

备份完成后可以在百度云客户端中看到我的应用数据目录,里面有你的备份文件。

2014-10-12_120731
QQ截图20150202004151
2014-10-12_120902

 

下面就开始撸代码了,首先是全自动获取accessToken到文件(可以根据手册自行获取,不过过程很麻烦),这里的BaiduPcs类是自己扩展的。

/**
	 * 获取pcs
	 * token并写入文件
	 */
	public function pcsTokenInitAction() {
		$callbackUrl = 'http://xxxxx.com/init';
		if (true === BaiduPcs::tokenInit ( $callbackUrl,APP_PATH.'/data/baiduPcsToken' )) {
			echo "OK";
		} else {
			echo "Not OK";
		}
	}

扩展的BaiduPcs,CLIENT_ID,CLIENT_SECRET 需要改成自己项目的。

<?php

namespace Ares;

use ErrorException;

require_once __DIR__ . '/../lib/BaiduPcs/libs/BaiduPCS.class.php';
/**
 * access
 * token
 * 文件保存在项目的data/pcs/token
 * 文件中
 *
 * @author Ares
 *        
 */
class BaiduPcs extends BaiduPCS {
	const CLIENT_ID = 'xxxx';
	const CLIENT_SECRET = 'xxxxxx';
	private $curl;
	function __construct($param = array('tockenFile'=>null)) {
		$this->curl = new CurlMulti ();
		if (! isset ( $param ['tokenFile'] )) {
			throw new ErrorException ( 'tokenFile not specified' );
		}
		$fileData = new FileData ( dirname ( $param ['tokenFile'] ), basename ( $param ['tokenFile'] ) );
		$token = $fileData->get ();
		if (! isset ( $token ) || empty ( $token )) {
			throw new ErrorException ( "token not found" );
		}
		$token = json_decode ( $token );
		$this->tokenCheck ( $token );
		$url = 'https://pcs.baidu.com/rest/2.0/pcs/quota?method=info&access_token=' . $token->access_token;
		$r = null;
		$this->curl->add ( array (
				'url' => $url 
		), function ($result) use(&$r) {
			if ($result ['info'] ['http_code'] == 200) {
				$r = $result;
			} else {
				throw new ErrorException ( 'http error, http_code=' . $result ['info'] ['http_code'] . ', url=' . $result ['info'] ['url'] );
			}
		}, function ($err) {
			throw new ErrorException ( 'curl error, ' . $err ['error'] [0] . ': ' . $err ['error'] [1] );
		} )->start ();
		$r = json_decode ( $r ['content'] );
		if (isset ( $r->error_code )) {
			if ($r->error_code == 111) {
				$token = json_decode ( $this->tokenRefresh ( $token->refresh_token ) );
			}
		}
		$this->tokenCheck ( $token );
		parent::__construct ( $token->access_token );
	}
	
	/**
	 * 检查一个对象形式的token是否可用,如果不可用程序退出
	 *
	 * @param unknown $token        	
	 */
	private function tokenCheck($token) {
		if (isset ( $token->error )) {
			$msg = "token is invalid, error=" . $token->error;
			if (isset ( $token->error_description )) {
				$msg .= ', ' . $token->error_description;
			}
			throw new ErrorException ( $msg );
		}
	}
	
	/**
	 * 返回项目的根路径
	 *
	 * @return string
	 */
	function getRootDir() {
		return '/apps/ares';
	}
	
	/**
	 * curl上传文件,默认的上传函数文件有多大就占用多大的内存
	 *
	 * @param unknown $file        	
	 * @return array false
	 */
	function curlUpload($file, $remoteFile) {
		if (! is_file ( $file )) {
			return false;
		}
		$size = filesize ( $file );
		if (0 == $size) {
			throw new ErrorException ( "file size is 0, file=" . $file );
		}
		$url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file?method=upload&path=' . urlencode ( $remoteFile ) . '&access_token=' . $this->getAccessToken ();
		$opt = array ();
		$timeout = $size / 1024 / 5;
		if ($timeout < 600) {
			$timeout = 600;
		}
		$opt [CURLOPT_TIMEOUT] = $timeout;
		$opt [CURLOPT_SSL_VERIFYPEER] = false;
		$opt [CURLOPT_SSL_VERIFYHOST] = false;
		$opt [CURLOPT_CONNECTTIMEOUT] = 30;
		$opt [CURLOPT_POST] = true;
		$opt [CURLOPT_POSTFIELDS] = array (
				'file' => '@' . $file 
		);
		$return = null;
		$this->curl->cbInfo = function ($info) use($file, $size) {
			$row = array_pop ( $info ['running'] );
			echo "r33[2K" . $row ['size_upload'] . '/' . $size . "t" . round ( $row ['size_upload'] / $size * 100, 2 ) . '% (' . round ( $row ['speed_upload'] / 1024, 0 ) . 'k/s)';
		};
		$this->curl->add ( array (
				'url' => $url,
				'opt' => $opt 
		), function ($r) use(&$return) {
			$return = $r;
		} )->start ();
		return $return;
	}
	
	/**
	 * 获得一个accesstoken,这个方法应该在controller的action中调用
	 *
	 * @param unknown $url
	 *        	调用此方法的url
	 */
	static function tokenInit($callbackUrl, $tokenFile) {
		if (! isset ( $_GET ['code'] )) {
			// 获取auth code
			$redirectUrl = "https://openapi.baidu.com/oauth/2.0/authorize?response_type=code&client_id=" . self::CLIENT_ID . "&redirect_uri=$callbackUrl&display=popup&scope=basic netdisk";
			header ( 'Location: ' . $redirectUrl, true, 302 );
		} else {
			// 根据auth code获取access token
			$url = "https://openapi.baidu.com/oauth/2.0/token?grant_type=authorization_code&code=$_GET[code]&client_id=" . self::CLIENT_ID . "&client_secret=" . self::CLIENT_SECRET . "&redirect_uri=$callbackUrl";
			$token = self::tokenSave ( $url, $tokenFile );
			$token = json_decode ( $token );
			if (isset ( $token->access_token )) {
				return true;
			}
		}
	}
	
	/**
	 * 用刷新token获取一个access
	 * token
	 */
	function tokenRefresh($refreshToken) {
		$url = "https://openapi.baidu.com/oauth/2.0/token?grant_type=refresh_token&refresh_token=$refreshToken&client_id=" . self::CLIENT_ID . "&client_secret=" . self::CLIENT_SECRET . "&scope=basic netdisk";
		return self::tokenSave ( $url );
	}
	
	/**
	 * 保存token到文件
	 *
	 * @param unknown $token        	
	 */
	private static function tokenSave($url, $tokenFile) {
		$token = null;
		$data = new FileData ( dirname ( $tokenFile ), basename ( $tokenFile ) );
		$curl = new CurlMulti ();
		$curl->add ( array (
				'url' => $url 
		), function ($r) use(&$token) {
			$token = $r ['content'];
		} )->start ();
		$data->put ( $token );
		return $token;
	}
}

实际的备份代码,把服务器要配置的文件列成数组,然后打包到~/backup目录,然后上传,上传完了删除,所有流程会写入一个文件,所以即使中断也没问题可以续传,MySQL导出是边导出边压缩的!,自动按日期创建目录,所有操作以最低的CPU优先级执行,所有功能和性能和易用性方面已经完全考虑,已经做到做好了!

<?php

namespace ControllerCli;

use LibController;
use ErrorException;
use AresFileData;
use AresBaiduPcs;
use AresLog;
use AresUtilHttp;

class Backup extends Controller {
	private $backupDir = '/root/backup';
	private $remoteDir;
	private $data;
	private $log;
	private $pcs;
	private $exePre = 'nice -n 19 ';
	function __construct() {
		if (PHP_SAPI != 'cli') {
			Http::error403 ();
		}
	}
	function init($new = false) {
		if (! is_dir ( $this->backupDir )) {
			if (false == mkdir ( $this->backupDir )) {
				throw new ErrorException ( 'failed to mkdir, dir=' . $this->backupDir );
			}
		}
		$this->data = new FileData ( APP_PATH . '/data', 'backup' );
		$this->log = new Log ( APP_PATH . '/log', 'backup' );
		if (null == $this->data->get () || $new) {
			$date = date ( 'Y-m-d' );
			$this->data->put ( array (
					'date' => $date,
					'list' => array () 
			) );
		} else {
			$date = $this->data->get ();
			$date = $date ['date'];
		}
		$this->pcs = new BaiduPcs ( array (
				'tokenFile' => APP_PATH . '/data/baiduPcsToken' 
		) );
		$this->remoteDir = $this->pcs->getRootDir () . '/vps/ramhost/' . $date;
		$this->pcs->makeDirectory ( $this->remoteDir );
	}
	
	/**
	 * 按顺序备份,如果中途中断,重新从中断的位置重新开始操作
	 */
	function indexAction($new = false) {
		$this->init ( $new );
		$this->tar ( 'cron.tar.gz', array (
				array (
						'/srv',
						'cron' 
				) 
		) );
		$this->tar ( 'etc.tar.gz', array (
				array (
						'/etc',
						'init.d' 
				),
				array (
						'/etc',
						'nginx',
						'cp' 
				),
				array (
						'/etc',
						'php.d' 
				),
				array (
						'/etc',
						'privoxy' 
				),
				array (
						'/etc',
						'php.ini' 
				),
				array (
						'/etc',
						'php-fpm.conf' 
				),
				array (
						'/etc',
						'php-fpm.d' 
				),
				array (
						'/etc',
						'my.cnf' 
				),
				array (
						'/etc',
						'postfix' 
				),
				array (
						'/etc',
						'dovecot.conf' 
				),
				array (
						'/usr/lib/sasl2/',
						'smtpd.conf' 
				),
				array (
						'/etc',
						'aliases' 
				),
				array (
						'/etc',
						'DIR_COLORS' 
				) 
		) );
		$this->tar ( 'crontab.tar.gz', array (
				array (
						'/var/spool',
						'cron' 
				) 
		) );
		$this->tar ( 'script.tar.gz', array (
				array (
						'/root',
						'script' 
				) 
		) );
		$this->tar ( 'blog.phpdr.net.tar.gz', array (
				array (
						'/srv/www_root',
						'blog.phpdr.net' 
				) 
		) );
		$this->tar ( 'svn.tar.gz', array (
				array (
						'/srv',
						'svn' 
				) 
		) );
		$this->tar ( 'tor.tar.gz', array (
				array (
						'/srv',
						'tor' 
				) 
		) );
		$this->tar ( 'privoxy.tar.gz', array (
				array (
						'/srv',
						'privoxy' 
				) 
		) );
		$this->tar ( 'bin.tar.gz', array (
				array (
						'/usr/local',
						'bin' 
				) 
		) );
		
		$mysqlHost = "127.0.0.1";
		$mysqlUser = "root";
		$mysqlPass = "xxxxx";
		/*
		 * blog
		 */
		$dbFile = 'blog.sql.gz';
		$dbData = $this->progress ( $dbFile );
		if (! isset ( $dbData ['backup'] ) || true != $dbData ['backup']) {
			echo `cd {$this->backupDir};{$this->exePre} mysqldump --events --ignore-table=mysql.event -h$mysqlHost -u$mysqlUser -p$mysqlPass blog |gzip -f > blog.sql.gz`;
		}
		$this->tar ( $dbFile );
		/*
		 * mysql
		 */
		$dbFile = 'mysql.sql.gz';
		$dbData = $this->progress ( $dbFile );
		if (! isset ( $dbData ['backup'] ) || true != $dbData ['backup']) {
			echo `cd {$this->backupDir};{$this->exePre} mysqldump --events --ignore-table=mysql.event -h$mysqlHost -u$mysqlUser -p$mysqlPass mysql | gzip -f > mysql.sql.gz`;
		}
		$this->tar ( $dbFile );
		
		/*
		 * 检查本轮上传是否全部成功
		 */
		$data = $this->progress ();
		$allTrue = true;
		foreach ( $data as $k => $v ) {
			if (true != $v ['backup'] || true != $v ['upload']) {
				$allTrue = false;
			}
		}
		if ($allTrue) {
			$this->data->unlink ();
		}
	}
	
	/**
	 * 打包文件并上传
	 *
	 * @param unknown $file        	
	 * @param unknown $files
	 *        	如果为null表示已经打包到指定目录
	 * @param
	 *        	string
	 *        	tar的-C参数
	 */
	private function tar($file, array $files = null) {
		$data = $this->progress ( $file );
		// 打包
		if (! isset ( $data ['backup'] ) || true != $data ['backup']) {
			if (isset ( $files )) {
				$str = '';
				$tmpFiles = array ();
				foreach ( $files as $v ) {
					// tar不能打包正在写入的文件,所以需要先拷贝
					if (isset ( $v [2] ) && $v [2] == 'cp') {
						`cp $v[0]/$v[1] -rf {$this->backupDir}/$v[1]`;
						$tmpFiles [] = $this->backupDir . '/' . $v [1];
						$v [0] = $this->backupDir;
					}
					$str .= "-C $v[0] $v[1] ";
				}
				$cmd = "rm -f $this->backupDir/$file;{$this->exePre} tar -czf $this->backupDir/$file $str";
				shell_exec ( $cmd );
				if (! empty ( $tmpFiles )) {
					foreach ( $tmpFiles as $v ) {
						`rm -rf $v`;
					}
				}
			}
			$data ['backup'] = true;
			$this->progress ( $file, $data );
		}
		// 上传
		if (! isset ( $data ['upload'] ) || true != $data ['upload']) {
			echo "n" . $file . "n";
			$r = $this->upload ( $file );
			$fail = true;
			if (isset ( $r ['content'] )) {
				if (! empty ( $r ['content'] )) {
					$r ['content'] = json_decode ( $r ['content'] );
					if (isset ( $r ['content']->fs_id ) || $r ['content']->error_code = 31061) {
						$data ['upload'] = true;
						$this->progress ( $file, $data );
						$fail = false;
					}
				}
			}
			if ($fail) {
				$this->log ( $r ['error_no'] . ' : ' . $r ['error_msg'] . ', file=' . $file );
			}
		}
		// 删除文件
		if (isset ( $data ['upload'] ) && true === $data ['upload']) {
			if (is_file ( $this->backupDir . '/' . $file )) {
				unlink ( $this->backupDir . '/' . $file );
			}
		}
	}
	
	/**
	 * 写入备份日志
	 *
	 * @param unknown $msg        	
	 */
	private function log($msg) {
		$this->log->append ( $msg );
	}
	
	/**
	 * 读取和写入进度文件
	 *
	 * @param string $file        	
	 * @param array $value
	 *        	包括
	 *        	backup和upload两项
	 * @return mixed
	 */
	private function progress($file = null, array $value = null) {
		static $data;
		if (! isset ( $data )) {
			$data = $this->data->get ();
		}
		if (! isset ( $file )) {
			return $data ['list'];
		} else {
			if (! isset ( $value )) {
				if (! isset ( $data ['list'] [$file] )) {
					$data ['list'] [$file] = array (
							'backup' => false,
							'upload' => false 
					);
				}
				return $data ['list'] [$file];
			} else {
				$data ['list'] [$file] = array_merge ( $data ['list'] [$file], $value );
				$this->data->put ( $data );
			}
		}
	}
	
	/**
	 * 备份文件到百度云
	 *
	 * @param unknown $file        	
	 */
	private function upload($file) {
		return $this->pcs->curlUpload ( $this->backupDir . '/' . $file, $this->remoteDir . '/' . $file );
	}
}

可能用到的写文件的类

<?php

namespace Ares;

use ErrorException;

class File {
	/**
	 * 利用堆栈清空目录
	 *
	 * @param string $dir
	 *        	目录名,最好是绝对路径,否则相对路径搞错了后果自负
	 * @return boolean
	 */
	static function dirFlush($dir) {
		clearstatcache ();
		// 格式化路径并清除结尾的斜线
		$dir = realpath ( $dir );
		if (file_exists ( $dir ) && is_writable ( $dir )) {
			// glob对点开头的文件处理有些问题,所以用scandir
			$files = scandir ( $dir );
			foreach ( $files as $k => $v ) {
				if ($v == '.' || $v == '..')
					unset ( $files [$k] );
				else
					$files [$k] = $dir . DIRECTORY_SEPARATOR . $v;
			}
			while ( ! empty ( $files ) ) {
				$file = array_pop ( $files );
				if (is_file ( $file )) {
					if (! unlink ( $file ))
						return false;
				} else {
					// 子目录为空就删除,否则进栈
					$dirFiles = scandir ( $file );
					if (count ( $dirFiles ) == 2) {
						if (! rmdir ( $file ))
							return false;
					} else {
						foreach ( $dirFiles as $k => $v ) {
							if ($v == '.' || $v == '..')
								unset ( $dirFiles [$k] );
							else
								$dirFiles [$k] = $file . DIRECTORY_SEPARATOR . $v;
						}
						$files = array_merge ( $files, array (
								$file 
						), $dirFiles );
					}
				}
			}
		}
		return false;
	}
	
	/**
	 * 在某个目录下循环创建子目录
	 *
	 * @param unknown $parent
	 *        	父路径
	 * @param unknown $dir
	 *        	相对路径
	 * @param number $mode
	 *        	模式
	 */
	static function mkDirSub($parent, $subdir, $mode = 0755) {
		if (! is_dir ( $parent )) {
			throw new ErrorException ( 'dir ' . $parent . ' doesn't exist' );
		}
		$parent = realpath ( $parent );
		$subdir = self::pathClean ( trim ( $subdir, ' /' ) );
		if (! empty ( $subdir )) {
			$subdirs = explode ( '/', $subdir );
			foreach ( $subdirs as $dir ) {
				$parent .= '/' . $dir;
				if (! file_exists ( $parent )) {
					mkdir ( $parent, $mode );
				}
			}
		}
	}
	
	/**
	 * 规范化一个相对路径或绝对路径,所有分隔符用/替换,计算路径中的.和..
	 *
	 * @param string $path        	
	 * @return string
	 */
	static function pathClean($path) {
		$path = trim ( $path );
		$path = str_replace ( '', '/', $path );
		$path = str_replace ( '', '/', $path );
		$path = str_replace ( '//', '/', $path );
		$arr = explode ( '/', $path );
		foreach ( $arr as $k => $v ) {
			if (empty ( $v ) || $v == '.')
				unset ( $arr [$k] );
			if ($v == '..') {
				unset ( $arr [$k] );
				prev ( $arr );
				list ( $k, $v ) = each ( $arr );
				unset ( $arr [$k] );
			}
		}
		$file = trim ( implode ( '/', $arr ), '/' );
		return $file;
	}
	
	/**
	 * 递归扫描目录
	 *
	 * @param string $dir
	 *        	被扫描的目录
	 * @param enum $mode
	 *        	文件类型,file,dir,null表示所有类型
	 * @param int $depth
	 *        	递归的深度,null是无限递归
	 * @param array $ignore
	 *        	通配符匹配
	 * @param number $order
	 *        	默认的排序顺序是按字母升序排列,如果设为 1,则按字母降序排列。
	 * @param string $context
	 *        	参见手册scandir
	 * @return array
	 */
	static function scandirR($dir, $mode = null, $depth = null, $ignore = array(), $order = 0, $context = null) {
		static $modes = array (
				'file',
				'dir',
				null 
		);
		$r = array ();
		if (! in_array ( $mode, $modes )) {
			throw new ErrorException ( 'mode is invalid' );
		}
		if (is_numeric ( $depth ) && -- $depth < 0)
			return $r;
		$dir = rtrim ( $dir, '/' );
		if (! is_dir ( $dir )) {
			return $r;
		}
		if (is_resource ( $context )) {
			$list = scandir ( $dir, $order, $context );
		} else {
			$list = scandir ( $dir, $order );
		}
		if (is_array ( $list ) and ! empty ( $list )) {
			foreach ( $list as $v ) {
				if ($v == '.' || $v == '..') {
					continue;
				} else {
					if (! empty ( $ignore )) {
						foreach ( $ignore as $v1 ) {
							if (fnmatch ( $v1, $v )) {
								continue 2;
							}
						}
					}
					if (is_file ( $dir . '/' . $v ) and ($mode == 'file' or $mode == null)) {
						$r [] = $v;
					} elseif (is_dir ( $dir . '/' . $v )) {
						if ($mode == 'dir' or $mode == null)
							$r [] = $v;
						$t = self::scandirR ( $dir . '/' . $v, $mode, $depth, $ignore, $order, $context );
						if (! empty ( $t )) {
							foreach ( $t as $k1 => $v1 ) {
								$t [$k1] = $v . '/' . $v1;
							}
						}
						$r = array_merge ( $r, $t );
					}
				}
			}
		} else {
			$r = $list;
		}
		return $r;
	}
}

 

<?php

namespace Ares;

use ErrorException;
use AresFile;

/**
 * 管理文件形式的数据。
 *
 * @author Ares
 *        
 */
class FileData {
	private $basePath;
	private $file;
	/**
	 *
	 * @param string $basePath        	
	 * @param unknown $path
	 *        	文件相对路径
	 */
	function __construct($basePath, $path, $suffix = 'dat') {
		$this->basePath = $basePath;
		$path = ltrim ( $path, ' /' );
		$this->file = $this->basePath . '/' . $path . '.' . $suffix;
		if (! is_dir ( dirname ( $this->file ) )) {
			File::mkDirSub ( $this->basePath, dirname ( $path ) );
		}
	}
	
	/**
	 * 写文件
	 *
	 * @param mixed $data        	
	 */
	function put($data) {
		$data = serialize ( $data );
		file_put_contents ( $this->file, $data, LOCK_EX );
	}
	
	/**
	 * 获取内容
	 */
	function get() {
		$file = $this->file;
		if (is_file ( $file )) {
			return unserialize ( file_get_contents ( $this->file ) );
		}
	}
	
	/**
	 * 数据文件修改时间
	 */
	function mtime() {
		$file = $this->file;
		if (is_file ( $file )) {
			return filemtime ( $file );
		} else {
			throw new ErrorException ( "file not found, file=" . $file );
		}
	}
	
	/**
	 * 删除数据文件
	 */
	function unlink() {
		$file = $this->file;
		if (is_file ( $file )) {
			return unlink ( $file );
		}
	}
}

 

发表评论

电子邮件地址不会被公开。

*