openresty实现图片（文件）服务器

文章目录

[隐藏]

• 介绍
  • 前序
  • 数据定义
• 代码实现
  • Nginx配置
  • lua脚本
  • Java
• 总结
  • 可能出现的问题
  • 开发中遇到的问题
  • 涉及到知识

介绍

前序

该功能是利用openresty的lua脚本实现的图片（文件）保存功能，文件上传使用java代码开发的

数据定义

上传数据和文件信息不分前后，但系统只会保存最后一对信息

• 数据格式：

{"fileDir":"文件保存的目录","fileName":"文件名"}  

• 返回结果

{"status":"是否成功","result":"返回结果","msg":"异常原因"}  enum status:["success","failed"]  

• 保存文件夹
  所保存到那个文件夹下，在nginx的perfix变量中定义

代码实现

Nginx配置

如下：

server {      listen       80;      server_name  localhost;  # 配置保存的文件夹      set $prefix "/data";        location /uploadimage {  # 配置是否每次lua更改都生效，适合调试时使用  #       lua_code_cache off;  # 配置lua脚本          content_by_lua_file /openresty-web/luascript/luascript;      }  # 用来配合理解传入到nginx的报文结构      location /uploadtest{  #       lua_code_cache off;          content_by_lua_file /openresty-web/luascript/luauploadtest;      }      error_page   500 502 503 504  /50x.html;      location = /50x.html {          root   html;      }  }  

lua脚本

luascript：

package.path = '/openresty-web/lualib/resty/?.lua;'  local upload = require "upload"  local cjson = require("cjson")    Result={status="success",result="",msg=""}  Result.__index=Result  function Result.conSuccess(ret)      ret["status"]="success"      ret["result"]="upload success"      return ret  end    function Result.conFailed(ret,err)      ret["status"]="failed"      ret["msg"]=err      ret["result"]="upload failed"      return ret  end    function Result:new()      local ret={}      setmetatable({},Result)      return ret  end    -- lua-resty-upload  local chunk_size = 4096  local form = upload:new(chunk_size)  if not form then      ngx.say(cjson.encode(Result.conFailed(Result:new(),"plase upload right info")))      return  end  local file  local filelen=0  form:set_timeout(0) -- 1 sec  local filename  local prefix=ngx.var.prefix    -- 匹配文件名，当前案例用于判断是否是文件模块  function get_filename(res)      local filename = ngx.re.match(res,'(.+)filename="(.+)"(.*)')      if filename then          return filename[2]      end  end      -- 用来开启输入流，当文件夹不存在时自动创建  function openstream(fileinfo,opt)      local file,err=io.open(prefix..fileinfo["fileDir"],"r")      if not file then          local start=string.find(err,"No such file or directory")          if start then              local exeret=os.execute("mkdir -p "..prefix..fileinfo["fileDir"])              if exeret ~= 0 then                  return nil,"Make directory failed"              end          else              return nil,err          end      end      file,err=io.open(prefix..fileinfo["fileDir"]..fileinfo["fileName"],opt)      return file,err  end    local osfilepath  local tmpfiletbl  local hasFile=false  local loopfile=false  local fileinfostr  local fileinfo  local result=Result:new()  -- 循环读取文件和文件信息  while true do      local typ, res, err = form:read()      if not typ then          break      end      if typ == "header" then          if res[1] ~= "Content-Type" then              filename = get_filename(res[2])              if filename then                  loopfile=true                  hasFile=true                  -- 判断是否有文件信息                  -- 如果没有记录内存                  if fileinfo then                      file,err=openstream(fileinfo,"w")                      if not file then                          break                      end                  else                      tmpfiletbl={}                  end              else                  loopfile = false                  fileinfostr = ""              end          end      end      if loopfile then          if typ == "body" then              if file then                  filelen= filelen + tonumber(string.len(res))                  file:write(res)              else                  table.insert(tmpfiletbl,res)              end          elseif typ == "part_end" then              if file then                  file:close()                  file = nil              end          end      else          if typ == "body" then              fileinfostr=fileinfostr .. res          elseif typ == "part_end" then              fileinfo = cjson.decode(fileinfostr)          end      end      if typ == "eof" then          break      end  end    if not hasFile then      err="plase upload file"  elseif not fileinfo or not fileinfo["fileDir"] or not fileinfo["fileName"] then      err="plase offer file info"  end    if err then      ngx.log(ngx.ERR,err)      Result.conFailed(result,err)      ngx.say(cjson.encode(result))      return  end    -- 因为有文件信息在文件之后传送的  -- 所以需要将输入到内存中的文件信息打印到磁盘  if tmpfiletbl and table.getn(tmpfiletbl) > 0 then      file,err=openstream(fileinfo,"w")      if not file then          ngx.log(ngx.ERR,err)          Result.conFailed(result,err)          ngx.say(cjson.encode(result))          return      else          for index,value in ipairs(tmpfiletbl)          do              filelen= filelen + tonumber(string.len(value))              file:write(value)          end          file:close()          file=nil      end  end      Result.conSuccess(result)  ngx.say(cjson.encode(result))  

luauploadtest：

local upload = require "resty.upload"  local cjson = require "cjson"    local chunk_size = 5 -- should be set to 4096 or 8192                       -- for real-world settings    local form, err = upload:new(chunk_size)  if not form then      ngx.log(ngx.ERR, "failed to new upload: ", err)      ngx.exit(500)  end    form:set_timeout(1000) -- 1 sec    while true do      local typ, res, err = form:read()      if not typ then          ngx.say("failed to read: ", err)          return      end        ngx.say("read: ", cjson.encode({typ, res}))        if typ == "eof" then          break      end  end    local typ, res, err = form:read()  ngx.say("read: ", cjson.encode({typ, res}))  

luauploadtest代码是官方提供代码

Java

ImageServer：

package cn.com.cgbchina.image;    import cn.com.cgbchina.image.exception.ImageDeleteException;  import cn.com.cgbchina.image.exception.ImageUploadException;  import org.springframework.web.multipart.MultipartFile;    /**   * Created by 11140721050130 on 16-3-22.   */  public interface ImageServer {      /**       * 刪除文件       *       * @param fileName 文件名       * @return 是否刪除成功       */      boolean delete(String fileName) throws ImageDeleteException;        /**       *       * @param originalName 原始文件名       * @param file 文件       * @return 文件上传后的相对路径       */      String upload(String originalName, MultipartFile file) throws ImageUploadException;  }  

LuaResult：

package cn.com.cgbchina.image.nginx;    import lombok.Getter;  import lombok.Setter;    /**   * Comment: 用来保存返回结果，   * 原本想放入到LuaImageServiceImpl的内部类中，   * 但是Jackson不支持，没法反序列化   * Created by ldaokun2006 on 2017/10/24.   */  @Setter  @Getter  public class LuaResult{      private LuaResultStatus status;      private String result;      private String msg;      private String httpUrl;      public LuaResult(){}        public void setStatus(String result){          status=LuaResultStatus.valueOf(result.toUpperCase());      }      public enum LuaResultStatus{          SUCCESS,FAILED;      }  }  

ImageServerImpl：

package cn.com.cgbchina.image.nginx;    import cn.com.cgbchina.common.utils.DateHelper;  import cn.com.cgbchina.image.ImageServer;  import cn.com.cgbchina.image.exception.ImageDeleteException;  import cn.com.cgbchina.image.exception.ImageUploadException;  import com.github.kevinsawicki.http.HttpRequest;  import com.google.common.base.Splitter;  import com.spirit.util.JsonMapper;  import lombok.AllArgsConstructor;  import lombok.Getter;  import lombok.Setter;  import lombok.extern.slf4j.Slf4j;  import org.springframework.stereotype.Service;  import org.springframework.web.multipart.MultipartFile;    import java.io.*;  import java.util.ArrayList;  import java.util.Date;  import java.util.List;  import java.util.UUID;  import java.util.concurrent.*;  import java.util.concurrent.atomic.AtomicInteger;    /**   * Comment: 实现文件上传功能   * Created by ldaokun2006 on 2017/10/16.   */  @Service  @Slf4j  public class LuaImageServiceImpl implements ImageServer{      // 存放nginx服务器url的，某些架构会有多个放置图片的地方      private List<String> httpUrls;      private ExecutorService fixedThreadPool ;      private Integer timeout;      private int threadSize=50;        public LuaImageServiceImpl(String httpUrls){          this(httpUrls,30000);      }        /**       *       * @param httpUrls 存放nginx服务器url       * @param timeout http超时时间       */      public LuaImageServiceImpl(String httpUrls,int timeout){          this.httpUrls=Splitter.on(";").splitToList(httpUrls);          // 没啥看得，就是想让线程池的名字易懂些          this.fixedThreadPool= new ThreadPoolExecutor(threadSize, threadSize,                  0L, TimeUnit.MILLISECONDS,                  new LinkedBlockingQueue<Runnable>(),new ThreadFactory(){                      private final AtomicInteger poolNumber = new AtomicInteger(1);                      private final ThreadGroup group;                      private final AtomicInteger threadNumber = new AtomicInteger(1);                      private final String namePrefix;                        {                          SecurityManager s = System.getSecurityManager();                          group = (s != null) ? s.getThreadGroup() :                                  Thread.currentThread().getThreadGroup();                          namePrefix = "LuaUploadPool-" +                                  poolNumber.getAndIncrement() +                                  "-thread-";                      }                        public Thread newThread(Runnable r) {                          Thread t = new Thread(group, r,                                  namePrefix + threadNumber.getAndIncrement(),                                  0);                          if (t.isDaemon())                              t.setDaemon(false);                          if (t.getPriority() != Thread.NORM_PRIORITY)                              t.setPriority(Thread.NORM_PRIORITY);                          return t;                      }                  });          this.timeout=timeout;      }        /**       * Comment: 没必要开发删除功能       * @param fileName 文件名       * @return       * @throws ImageDeleteException       */      @Override      public boolean delete(String fileName) throws ImageDeleteException {          return true;      }        /**       * Commont: 用来给SpringMVC用       * @param originalName 原始文件名       * @param file 文件       * @return       * @throws ImageUploadException       */      @Override      public String upload(String originalName, MultipartFile file) throws ImageUploadException {          try {              return this.upload(originalName,file.getInputStream());          } catch (IOException e) {              log.error("upload fail : " + e.getMessage(), e);              throw new ImageUploadException("upload fail : "+e.getMessage(),e);          }      }        /**       * Commont: 上传图片核心代码       * @param originalName 原始文件名       * @param inputStream 要上传文件的文件流       * @return       * @throws ImageUploadException       */      private String upload(String originalName,InputStream inputStream) throws ImageUploadException {          ByteArrayOutputStream byteOutStream = null;          try {              //准备数据              byte[] tmpData=new byte[1024];              byte[] inputData;              byteOutStream = new ByteArrayOutputStream();              int len=0;              while((len=inputStream.read(tmpData,0,tmpData.length))!=-1){                  byteOutStream.write(tmpData,0,len);              }              inputData=byteOutStream.toByteArray();              LuaSend sendInfo = new LuaSend(generateFileDir(),generateFileName(originalName));              List<Future<LuaResult>> resultList=new ArrayList<>(httpUrls.size());                //发送图片              for(String httpUrl:httpUrls) {                  SendImg sendImg = new SendImg(httpUrl,sendInfo, inputData,this.timeout);                  resultList.add(fixedThreadPool.submit(sendImg));              }              for(Future<LuaResult> future:resultList) {                  // 线程池异常在这里抛出                  LuaResult resultLuaResult = future.get();                  if (LuaResult.LuaResultStatus.SUCCESS != resultLuaResult.getStatus()) {                      throw new ImageUploadException("lua result url:"+resultLuaResult.getHttpUrl()+" msg : " + resultLuaResult.getMsg());                  }              }                return sendInfo.toString();          }catch (Exception e){              log.error("upload fail : "+e.getMessage(),e);              throw new ImageUploadException("upload fail : "+e.getMessage(),e);          }finally {              try {                  if(byteOutStream!=null) {                      byteOutStream.close();                  }                  if(inputStream!=null) {                      inputStream.close();                  }              } catch (IOException e) {                  throw new ImageUploadException("upload fail : "+e.getMessage(),e);              }          }      }      String separator=File.separator;      String dateFormat=separator+"yyyy"+separator+"MM"+separator+"dd"+ separator;        /**       * Comment：根据时间做路径，防止某一个文件夹东西太多       * @return 返回要保存的路径       */      private String generateFileDir(){          return DateHelper.date2string(new Date(),dateFormat);      }        /**       * Comment: 用UUID防止文件名重复       * @param originalName 源文件名字       * @return 要保存的文件名       */      private String generateFileName(String originalName){          return UUID.randomUUID().toString();      }        /**       * Comment: 用来发送图片的       */      @AllArgsConstructor      class SendImg implements  Callable<LuaResult>{            private String httpUrl;          private LuaSend sendInfo;          private byte[] inputStream;          private Integer timeout;              @Override          public LuaResult call() throws Exception {              try {                  String resultStr = HttpRequest                          .post(httpUrl, false)                          .part("fileInfo", JsonMapper.JSON_NON_EMPTY_MAPPER.toJson(sendInfo))                          // 这个地方有个坑，part上传图片必须要用这个方式，                          // 不能用没有Content-Type和fileName的                          .part("file", sendInfo.getFileName(), "multipart/form-data; boundary=00content0boundary00", new ByteArrayInputStream(inputStream))                          .connectTimeout(timeout).body();                  log.info("result:"+resultStr);                  LuaResult result = JsonMapper.JSON_NON_DEFAULT_MAPPER.fromJson(resultStr, LuaResult.class);                  result.setHttpUrl(httpUrl);                  return result;              }catch(Exception e){                  throw new ImageUploadException("upload failed url:"+httpUrl+" info:"+sendInfo.toString(),e);              }          }      }        /**       * Comment：文件数据       */      @Setter      @Getter      @AllArgsConstructor      class LuaSend {          // 文件目录          private String fileDir;          // 文件名          private String fileName;          @Override          public String toString(){              return fileDir+fileName;          }      }          /**       * Comment：测试用       * @param args       * @throws ImageUploadException       * @throws FileNotFoundException       */      public static void main(String[] args) throws ImageUploadException, FileNotFoundException {          LuaImageServiceImpl service=new LuaImageServiceImpl("http://192.168.99.102/uploadimage");          try {              System.out.println(service.upload("qqqqq", new FileInputStream("D:\shsh.txt")));          }finally {              service.fixedThreadPool.shutdown();          }      }  }  

总结

可能出现的问题

1. 上传两个图片或图片信息时系统只保留最后一个信息
2. 图片和图片信息可以随意放置，但是这两个必须成对发送，建议先发送图片信息后发送图片，这样图片不用在lua处保存到内存中
3. 上传大图片时会出现文件太大的提示，需要在nginx配置文件中添加client_max_body_size 100M;
4. Http Header的Content-Type必须使用multipart/form-data;
   boundary=00content0boundary00，boundary必须存在不然不好用
5. 传送图片HttpRequest.part上传图片必须写明Content-type和fileName，不然不好用但是Content-type不用非的用例子上的方式
6. 图片信息必须拷贝成byte型，因为多线程使用时需要各自发送

开发中遇到的问题

1. 传送图片HttpRequest.part上传图片必须写明Content-type，不然不好用
2. Jackson和fastjson对于需要反序列化的类，必须有无参构造函数，并且不能是内部类
3. lua的string.find如果没有找到，返回结果为nil
4. CSDN的编辑器，无需功能不好用

涉及到知识

1. HttpRequest.part用来上传Content-type:multipart/form-data;
2. lua的使用：http://www.runoob.com/lua/lua-tutorial.html
3. openresty的api：http://openresty.org/cn/components.html

原文出处：csdn -> https://blog.csdn.net/ldaokun2006/article/details/78333622