Linux系统运维日志

介绍

前序

该功能是利用openresty的lua脚本实现的图片（文件）保存功能，文件上传使用java代码开发的

数据定义

上传数据和文件信息不分前后，但系统只会保存最后一对信息

• 数据格式：

{"fileDir":"文件保存的目录","fileName":"文件名"}

• 返回结果

{"status":"是否成功","result":"返回结果","msg":"异常原因"}
enum status:["success","failed"]

• 保存文件夹
  所保存到那个文件夹下，在nginx的perfix变量中定义

代码实现

Nginx配置

如下：

server {
    listen       80;
    server_name  localhost;
# 配置保存的文件夹
    set $prefix "/data";

    location /uploadimage {
# 配置是否每次lua更改都生效，适合调试时使用
#       lua_code_cache off;
# 配置lua脚本
        content_by_lua_file /openresty-web/luascript/luascript;
    }
# 用来配合理解传入到nginx的报文结构
    location /uploadtest{
#       lua_code_cache off;
        content_by_lua_file /openresty-web/luascript/luauploadtest;
    }
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   html;
    }
}

lua脚本

luascript：

package.path = '/openresty-web/lualib/resty/?.lua;'
local upload = require "upload"
local cjson = require("cjson")

Result={status="success",result="",msg=""}
Result.__index=Result
function Result.conSuccess(ret)
    ret["status"]="success"
    ret["result"]="upload success"
    return ret
end

function Result.conFailed(ret,err)
    ret["status"]="failed"
    ret["msg"]=err
    ret["result"]="upload failed"
    return ret
end

function Result:new()
    local ret={}
    setmetatable({},Result)
    return ret
end

-- lua-resty-upload
local chunk_size = 4096
local form = upload:new(chunk_size)
if not form then
    ngx.say(cjson.encode(Result.conFailed(Result:new(),"plase upload right info")))
    return 
end
local file
local filelen=0
form:set_timeout(0) -- 1 sec
local filename
local prefix=ngx.var.prefix

-- 匹配文件名，当前案例用于判断是否是文件模块
function get_filename(res)
    local filename = ngx.re.match(res,'(.+)filename="(.+)"(.*)')
    if filename then 
        return filename[2]
    end
end


-- 用来开启输入流，当文件夹不存在时自动创建
function openstream(fileinfo,opt)
    local file,err=io.open(prefix..fileinfo["fileDir"],"r")
    if not file then
        local start=string.find(err,"No such file or directory")
        if start then
            local exeret=os.execute("mkdir -p "..prefix..fileinfo["fileDir"])
            if exeret ~= 0 then
                return nil,"Make directory failed"
            end
        else
            return nil,err
        end
    end
    file,err=io.open(prefix..fileinfo["fileDir"]..fileinfo["fileName"],opt)
    return file,err
end

local osfilepath
local tmpfiletbl
local hasFile=false
local loopfile=false
local fileinfostr
local fileinfo
local result=Result:new()
-- 循环读取文件和文件信息
while true do
    local typ, res, err = form:read()
    if not typ then
        break
    end
    if typ == "header" then
        if res[1] ~= "Content-Type" then
            filename = get_filename(res[2])
            if filename then
                loopfile=true
                hasFile=true
                -- 判断是否有文件信息
                -- 如果没有记录内存
                if fileinfo then
                    file,err=openstream(fileinfo,"w")
                    if not file then
                        break
                    end
                else
                    tmpfiletbl={}
                end
            else
                loopfile = false
                fileinfostr = ""
            end
        end
    end
    if loopfile then
        if typ == "body" then
            if file then
                filelen= filelen + tonumber(string.len(res))    
                file:write(res)
            else
                table.insert(tmpfiletbl,res)
            end
        elseif typ == "part_end" then
            if file then
                file:close()
                file = nil
            end
        end
    else
        if typ == "body" then
            fileinfostr=fileinfostr .. res
        elseif typ == "part_end" then
            fileinfo = cjson.decode(fileinfostr)
        end
    end
    if typ == "eof" then
        break
    end
end

if not hasFile then
    err="plase upload file"
elseif not fileinfo or not fileinfo["fileDir"] or not fileinfo["fileName"] then
    err="plase offer file info"
end

if err then
    ngx.log(ngx.ERR,err)
    Result.conFailed(result,err)
    ngx.say(cjson.encode(result))
    return 
end

-- 因为有文件信息在文件之后传送的
-- 所以需要将输入到内存中的文件信息打印到磁盘
if tmpfiletbl and table.getn(tmpfiletbl) > 0 then
    file,err=openstream(fileinfo,"w")
    if not file then
        ngx.log(ngx.ERR,err)
        Result.conFailed(result,err)
        ngx.say(cjson.encode(result))
        return 
    else
        for index,value in ipairs(tmpfiletbl)
        do
            filelen= filelen + tonumber(string.len(value)) 
            file:write(value)
        end
        file:close()
        file=nil
    end
end


Result.conSuccess(result)
ngx.say(cjson.encode(result))

luauploadtest：

local upload = require "resty.upload"
local cjson = require "cjson"

local chunk_size = 5 -- should be set to 4096 or 8192
                     -- for real-world settings

local form, err = upload:new(chunk_size)
if not form then
    ngx.log(ngx.ERR, "failed to new upload: ", err)
    ngx.exit(500)
end

form:set_timeout(1000) -- 1 sec

while true do
    local typ, res, err = form:read()
    if not typ then
        ngx.say("failed to read: ", err)
        return
    end

    ngx.say("read: ", cjson.encode({typ, res}))

    if typ == "eof" then
        break
    end
end

local typ, res, err = form:read()
ngx.say("read: ", cjson.encode({typ, res}))

luauploadtest代码是官方提供代码

Java

ImageServer：

package cn.com.cgbchina.image;

import cn.com.cgbchina.image.exception.ImageDeleteException;
import cn.com.cgbchina.image.exception.ImageUploadException;
import org.springframework.web.multipart.MultipartFile;

/**
 * Created by 11140721050130 on 16-3-22.
 */
public interface ImageServer {
    /**
     * 刪除文件
     *
     * @param fileName 文件名
     * @return 是否刪除成功
     */
    boolean delete(String fileName) throws ImageDeleteException;

    /**
     *
     * @param originalName 原始文件名
     * @param file 文件
     * @return 文件上传后的相对路径
     */
    String upload(String originalName, MultipartFile file) throws ImageUploadException;
}

LuaResult：

package cn.com.cgbchina.image.nginx;

import lombok.Getter;
import lombok.Setter;

/**
 * Comment: 用来保存返回结果，
 * 原本想放入到LuaImageServiceImpl的内部类中，
 * 但是Jackson不支持，没法反序列化
 * Created by ldaokun2006 on 2017/10/24.
 */
@Setter
@Getter
public class LuaResult{
    private LuaResultStatus status;
    private String result;
    private String msg;
    private String httpUrl;
    public LuaResult(){}

    public void setStatus(String result){
        status=LuaResultStatus.valueOf(result.toUpperCase());
    }
    public enum LuaResultStatus{
        SUCCESS,FAILED;
    }
}

ImageServerImpl：

package cn.com.cgbchina.image.nginx;

import cn.com.cgbchina.common.utils.DateHelper;
import cn.com.cgbchina.image.ImageServer;
import cn.com.cgbchina.image.exception.ImageDeleteException;
import cn.com.cgbchina.image.exception.ImageUploadException;
import com.github.kevinsawicki.http.HttpRequest;
import com.google.common.base.Splitter;
import com.spirit.util.JsonMapper;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Comment: 实现文件上传功能
 * Created by ldaokun2006 on 2017/10/16.
 */
@Service
@Slf4j
public class LuaImageServiceImpl implements ImageServer{
    // 存放nginx服务器url的，某些架构会有多个放置图片的地方
    private List<String> httpUrls;
    private ExecutorService fixedThreadPool ;
    private Integer timeout;
    private int threadSize=50;

    public LuaImageServiceImpl(String httpUrls){
        this(httpUrls,30000);
    }

    /**
     *
     * @param httpUrls 存放nginx服务器url
     * @param timeout http超时时间
     */
    public LuaImageServiceImpl(String httpUrls,int timeout){
        this.httpUrls=Splitter.on(";").splitToList(httpUrls);
        // 没啥看得，就是想让线程池的名字易懂些
        this.fixedThreadPool= new ThreadPoolExecutor(threadSize, threadSize,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>(),new ThreadFactory(){
                    private final AtomicInteger poolNumber = new AtomicInteger(1);
                    private final ThreadGroup group;
                    private final AtomicInteger threadNumber = new AtomicInteger(1);
                    private final String namePrefix;

                    {
                        SecurityManager s = System.getSecurityManager();
                        group = (s != null) ? s.getThreadGroup() :
                                Thread.currentThread().getThreadGroup();
                        namePrefix = "LuaUploadPool-" +
                                poolNumber.getAndIncrement() +
                                "-thread-";
                    }

                    public Thread newThread(Runnable r) {
                        Thread t = new Thread(group, r,
                                namePrefix + threadNumber.getAndIncrement(),
                                0);
                        if (t.isDaemon())
                            t.setDaemon(false);
                        if (t.getPriority() != Thread.NORM_PRIORITY)
                            t.setPriority(Thread.NORM_PRIORITY);
                        return t;
                    }
                });
        this.timeout=timeout;
    }

    /**
     * Comment: 没必要开发删除功能
     * @param fileName 文件名
     * @return
     * @throws ImageDeleteException
     */
    @Override
    public boolean delete(String fileName) throws ImageDeleteException {
        return true;
    }

    /**
     * Commont: 用来给SpringMVC用
     * @param originalName 原始文件名
     * @param file 文件
     * @return
     * @throws ImageUploadException
     */
    @Override
    public String upload(String originalName, MultipartFile file) throws ImageUploadException {
        try {
            return this.upload(originalName,file.getInputStream());
        } catch (IOException e) {
            log.error("upload fail : " + e.getMessage(), e);
            throw new ImageUploadException("upload fail : "+e.getMessage(),e);
        }
    }

    /**
     * Commont: 上传图片核心代码
     * @param originalName 原始文件名
     * @param inputStream 要上传文件的文件流
     * @return
     * @throws ImageUploadException
     */
    private String upload(String originalName,InputStream inputStream) throws ImageUploadException {
        ByteArrayOutputStream byteOutStream = null;
        try {
            //准备数据
            byte[] tmpData=new byte[1024];
            byte[] inputData;
            byteOutStream = new ByteArrayOutputStream();
            int len=0;
            while((len=inputStream.read(tmpData,0,tmpData.length))!=-1){
                byteOutStream.write(tmpData,0,len);
            }
            inputData=byteOutStream.toByteArray();
            LuaSend sendInfo = new LuaSend(generateFileDir(),generateFileName(originalName));
            List<Future<LuaResult>> resultList=new ArrayList<>(httpUrls.size());

            //发送图片
            for(String httpUrl:httpUrls) {
                SendImg sendImg = new SendImg(httpUrl,sendInfo, inputData,this.timeout);
                resultList.add(fixedThreadPool.submit(sendImg));
            }
            for(Future<LuaResult> future:resultList) {
                // 线程池异常在这里抛出
                LuaResult resultLuaResult = future.get();
                if (LuaResult.LuaResultStatus.SUCCESS != resultLuaResult.getStatus()) {
                    throw new ImageUploadException("lua result url:"+resultLuaResult.getHttpUrl()+" msg : " + resultLuaResult.getMsg());
                }
            }

            return sendInfo.toString();
        }catch (Exception e){
            log.error("upload fail : "+e.getMessage(),e);
            throw new ImageUploadException("upload fail : "+e.getMessage(),e);
        }finally {
            try {
                if(byteOutStream!=null) {
                    byteOutStream.close();
                }
                if(inputStream!=null) {
                    inputStream.close();
                }
            } catch (IOException e) {
                throw new ImageUploadException("upload fail : "+e.getMessage(),e);
            }
        }
    }
    String separator=File.separator;
    String dateFormat=separator+"yyyy"+separator+"MM"+separator+"dd"+ separator;

    /**
     * Comment：根据时间做路径，防止某一个文件夹东西太多
     * @return 返回要保存的路径
     */
    private String generateFileDir(){
        return DateHelper.date2string(new Date(),dateFormat);
    }

    /**
     * Comment: 用UUID防止文件名重复
     * @param originalName 源文件名字
     * @return 要保存的文件名
     */
    private String generateFileName(String originalName){
        return UUID.randomUUID().toString();
    }

    /**
     * Comment: 用来发送图片的
     */
    @AllArgsConstructor
    class SendImg implements  Callable<LuaResult>{

        private String httpUrl;
        private LuaSend sendInfo;
        private byte[] inputStream;
        private Integer timeout;


        @Override
        public LuaResult call() throws Exception {
            try {
                String resultStr = HttpRequest
                        .post(httpUrl, false)
                        .part("fileInfo", JsonMapper.JSON_NON_EMPTY_MAPPER.toJson(sendInfo))
                        // 这个地方有个坑，part上传图片必须要用这个方式，
                        // 不能用没有Content-Type和fileName的
                        .part("file", sendInfo.getFileName(), "multipart/form-data; boundary=00content0boundary00", new ByteArrayInputStream(inputStream))
                        .connectTimeout(timeout).body();
                log.info("result:"+resultStr);
                LuaResult result = JsonMapper.JSON_NON_DEFAULT_MAPPER.fromJson(resultStr, LuaResult.class);
                result.setHttpUrl(httpUrl);
                return result;
            }catch(Exception e){
                throw new ImageUploadException("upload failed url:"+httpUrl+" info:"+sendInfo.toString(),e);
            }
        }
    }

    /**
     * Comment：文件数据
     */
    @Setter
    @Getter
    @AllArgsConstructor
    class LuaSend {
        // 文件目录
        private String fileDir;
        // 文件名
        private String fileName;
        @Override
        public String toString(){
            return fileDir+fileName;
        }
    }


    /**
     * Comment：测试用
     * @param args
     * @throws ImageUploadException
     * @throws FileNotFoundException
     */
    public static void main(String[] args) throws ImageUploadException, FileNotFoundException {
        LuaImageServiceImpl service=new LuaImageServiceImpl("http://192.168.99.102/uploadimage");
        try {
            System.out.println(service.upload("qqqqq", new FileInputStream("D:\shsh.txt")));
        }finally {
            service.fixedThreadPool.shutdown();
        }
    }
}

总结

可能出现的问题

1. 上传两个图片或图片信息时系统只保留最后一个信息
2. 图片和图片信息可以随意放置，但是这两个必须成对发送，建议先发送图片信息后发送图片，这样图片不用在lua处保存到内存中
3. 上传大图片时会出现文件太大的提示，需要在nginx配置文件中添加client_max_body_size 100M;
4. Http Header的Content-Type必须使用multipart/form-data;
   boundary=00content0boundary00，boundary必须存在不然不好用
5. 传送图片HttpRequest.part上传图片必须写明Content-type和fileName，不然不好用但是Content-type不用非的用例子上的方式
6. 图片信息必须拷贝成byte型，因为多线程使用时需要各自发送

开发中遇到的问题

1. 传送图片HttpRequest.part上传图片必须写明Content-type，不然不好用
2. Jackson和fastjson对于需要反序列化的类，必须有无参构造函数，并且不能是内部类
3. lua的string.find如果没有找到，返回结果为nil
4. CSDN的编辑器，无需功能不好用

涉及到知识

1. HttpRequest.part用来上传Content-type:multipart/form-data;
2. lua的使用：http://www.runoob.com/lua/lua-tutorial.html
3. openresty的api：http://openresty.org/cn/components.html