xrootd
XrdClZipArchive.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_
26 #define SRC_XRDZIP_XRDZIPARCHIVE_HH_
27 
28 #include "XrdCl/XrdClFile.hh"
30 #include "XrdCl/XrdClJobManager.hh"
31 #include "XrdCl/XrdClDefaultEnv.hh"
32 #include "XrdCl/XrdClPostMaster.hh"
33 #include "XrdZip/XrdZipEOCD.hh"
34 #include "XrdZip/XrdZipCDFH.hh"
36 #include "XrdZip/XrdZipLFH.hh"
37 #include "XrdCl/XrdClZipCache.hh"
38 
39 #include <memory>
40 #include <unordered_map>
41 
42 //-----------------------------------------------------------------------------
43 // Forward declaration needed for friendship
44 //-----------------------------------------------------------------------------
45 namespace XrdEc{ class StrmWriter; class Reader; template<bool> class OpenOnlyImpl; };
46 class MicroTest;
47 
48 namespace XrdCl
49 {
50 
51  using namespace XrdZip;
52 
53  //---------------------------------------------------------------------------
54  // ZipArchive provides following functionalities:
55  // - parsing of existing ZIP archive
56  // - reading data from existing ZIP archive
57  // - appending data to existing ZIP archive
58  // - querying stat info and checksum for given file in ZIP archive
59  //---------------------------------------------------------------------------
60  class ZipArchive
61  {
62  friend class XrdEc::StrmWriter;
63  friend class XrdEc::Reader;
64  template<bool>
65  friend class XrdEc::OpenOnlyImpl;
66  friend class ::MicroTest;
67 
68  public:
69  //-----------------------------------------------------------------------
71  //-----------------------------------------------------------------------
72  ZipArchive( bool enablePlugIns = true );
73 
74  //-----------------------------------------------------------------------
76  //-----------------------------------------------------------------------
77  virtual ~ZipArchive();
78 
79  //-----------------------------------------------------------------------
87  //-----------------------------------------------------------------------
88  XRootDStatus OpenArchive( const std::string &url,
89  OpenFlags::Flags flags,
90  ResponseHandler *handler,
91  uint16_t timeout = 0 );
92 
93  //-----------------------------------------------------------------------
101  //-----------------------------------------------------------------------
102  XRootDStatus OpenFile( const std::string &fn,
104  uint64_t size = 0,
105  uint32_t crc32 = 0 );
106 
107  //-----------------------------------------------------------------------
116  //-----------------------------------------------------------------------
117  inline
118  XRootDStatus Read( uint64_t offset,
119  uint32_t size,
120  void *buffer,
121  ResponseHandler *handler,
122  uint16_t timeout = 0 )
123  {
124  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
125  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
126  }
127 
128  //-----------------------------------------------------------------------
138  //-----------------------------------------------------------------------
139  XRootDStatus ReadFrom( const std::string &fn,
140  uint64_t offset,
141  uint32_t size,
142  void *buffer,
143  ResponseHandler *handler,
144  uint16_t timeout = 0 );
145 
146  //-----------------------------------------------------------------------
154  //-----------------------------------------------------------------------
155  inline XRootDStatus Write( uint32_t size,
156  const void *buffer,
157  ResponseHandler *handler,
158  uint16_t timeout = 0 )
159  {
160  if( openstage != Done || openfn.empty() )
161  return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
162 
163  return WriteImpl( size, buffer, handler, timeout );
164  }
165 
166  //-----------------------------------------------------------------------
171  //-----------------------------------------------------------------------
172  XRootDStatus UpdateMetadata( uint32_t crc32 );
173 
174  //-----------------------------------------------------------------------
184  //-----------------------------------------------------------------------
185  XRootDStatus AppendFile( const std::string &fn,
186  uint32_t crc32,
187  uint32_t size,
188  const void *buffer,
189  ResponseHandler *handler,
190  uint16_t timeout = 0 );
191 
192  //-----------------------------------------------------------------------
198  //-----------------------------------------------------------------------
199  inline XRootDStatus Stat( const std::string &fn, StatInfo *&info )
200  { // make sure archive has been opened and CD has been parsed
201  if( openstage != Done )
202  return XRootDStatus( stError, errInvalidOp );
203  // make sure the file is part of the archive
204  auto cditr = cdmap.find( fn );
205  if( cditr == cdmap.end() )
206  return XRootDStatus( stError, errNotFound );
207  // create the result
208  info = make_stat( fn );
209  return XRootDStatus();
210  }
211 
212  //-----------------------------------------------------------------------
217  //-----------------------------------------------------------------------
218  inline XRootDStatus Stat( StatInfo *&info )
219  {
220  if( openfn.empty() )
221  return XRootDStatus( stError, errInvalidOp );
222  return Stat( openfn, info );
223  }
224 
225  //-----------------------------------------------------------------------
231  //-----------------------------------------------------------------------
232  inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum )
233  { // make sure archive has been opened and CD has been parsed
234  if( openstage != Done )
235  return XRootDStatus( stError, errInvalidOp );
236  // make sure the file is part of the archive
237  auto cditr = cdmap.find( fn );
238  if( cditr == cdmap.end() )
239  return XRootDStatus( stError, errNotFound );
240  cksum = cdvec[cditr->second]->ZCRC32;
241  return XRootDStatus();
242  }
243 
244  //-----------------------------------------------------------------------
246  //
250  //-----------------------------------------------------------------------
252  uint16_t timeout = 0 );
253 
254  //-----------------------------------------------------------------------
257  //-----------------------------------------------------------------------
259  {
260  if( openstage != Done || openfn.empty() )
262  0, "Archive not opened." );
263  openfn.clear();
264  lfh.reset();
265  return XRootDStatus();
266  }
267 
268  //-----------------------------------------------------------------------
271  //-----------------------------------------------------------------------
272  XRootDStatus List( DirectoryList *&list );
273 
274  //-----------------------------------------------------------------------
276  //-----------------------------------------------------------------------
277  inline bool IsOpen()
278  {
279  return openstage == Done;
280  }
281 
282  //-----------------------------------------------------------------------
284  //-----------------------------------------------------------------------
285  inline bool SetProperty( const std::string &name, const std::string &value )
286  {
287  return archive.SetProperty( name, value );
288  }
289 
290  //-----------------------------------------------------------------------
292  //-----------------------------------------------------------------------
293  inline bool GetProperty( const std::string &name, std::string &value )
294  {
295  return archive.GetProperty( name, value );
296  }
297 
298  private:
299 
300  //-----------------------------------------------------------------------
309  //-----------------------------------------------------------------------
310  XRootDStatus WriteImpl( uint32_t size,
311  const void *buffer,
312  ResponseHandler *handler,
313  uint16_t timeout );
314 
315  //-----------------------------------------------------------------------
323  //-----------------------------------------------------------------------
324  XRootDStatus OpenOnly( const std::string &url,
325  bool update,
326  ResponseHandler *handler,
327  uint16_t timeout = 0 );
328 
329  //-----------------------------------------------------------------------
333  //-----------------------------------------------------------------------
334  buffer_t GetCD();
335 
336  //-----------------------------------------------------------------------
340  //-----------------------------------------------------------------------
341  void SetCD( const buffer_t &buffer );
342 
343  //-----------------------------------------------------------------------
348  //-----------------------------------------------------------------------
349  template<typename Response>
350  inline static AnyObject* PkgRsp( Response *rsp )
351  {
352  if( !rsp ) return nullptr;
353  AnyObject *pkg = new AnyObject();
354  pkg->Set( rsp );
355  return pkg;
356  }
357 
358  //-----------------------------------------------------------------------
360  //-----------------------------------------------------------------------
361  template<typename Response>
362  inline static void Free( XRootDStatus *st, Response *rsp )
363  {
364  delete st;
365  delete rsp;
366  }
367 
368  //-----------------------------------------------------------------------
375  //-----------------------------------------------------------------------
376  template<typename Response>
377  inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr )
378  {
379  if( !handler ) return Free( st, rsp );
380  ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 );
382  }
383 
384  //-----------------------------------------------------------------------
390  //-----------------------------------------------------------------------
391  inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size )
392  {
393  StatInfo *info = new StatInfo( starch );
394  uint32_t flags = info->GetFlags();
395  info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable
396  info->SetSize( size );
397  return info;
398  }
399 
400  //-----------------------------------------------------------------------
405  //-----------------------------------------------------------------------
406  inline StatInfo* make_stat( const std::string &fn )
407  {
408  StatInfo *infoptr = 0;
409  XRootDStatus st = archive.Stat( false, infoptr );
410  std::unique_ptr<StatInfo> stinfo( infoptr );
411  auto itr = cdmap.find( fn );
412  if( itr == cdmap.end() ) return nullptr;
413  size_t index = itr->second;
414  return make_stat( *stinfo, cdvec[index]->uncompressedSize );
415  }
416 
417  //-----------------------------------------------------------------------
419  //-----------------------------------------------------------------------
420  inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() )
421  {
422  return new XRootDStatus( status );
423  }
424 
425  //-----------------------------------------------------------------------
427  //-----------------------------------------------------------------------
428  inline void Clear()
429  {
430  buffer.reset();
431  eocd.reset();
432  cdvec.clear();
433  cdmap.clear();
434  zip64eocd.reset();
435  openstage = None;
436  }
437 
438  //-----------------------------------------------------------------------
440  //-----------------------------------------------------------------------
442  {
443  None = 0, //< opening/parsing not started
444  HaveEocdBlk, //< we have the End of Central Directory record
445  HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record
446  HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record
447  HaveCdRecords, //< we have Central Directory records
448  Done, //< we are done parsing the Central Directory
449  Error, //< opening/parsing failed
450  NotParsed //< the ZIP archive has been opened but Central Directory is not parsed
451  };
452 
453  //-----------------------------------------------------------------------
455  //-----------------------------------------------------------------------
456  struct NewFile
457  {
458  NewFile( uint64_t offset, std::unique_ptr<LFH> lfh ) : offset( offset ),
459  lfh( std::move( lfh ) ),
460  overwrt( false )
461  {
462  }
463 
464  NewFile( NewFile && nf ) : offset( nf.offset ),
465  lfh( std::move( nf.lfh ) ),
466  overwrt( nf.overwrt )
467  {
468  }
469 
470  uint64_t offset; // the offset of the LFH of the file
471  std::unique_ptr<LFH> lfh; // LFH of the file
472  bool overwrt; // if true the LFH needs to be overwritten on close
473  };
474 
475  //-----------------------------------------------------------------------
477  //-----------------------------------------------------------------------
478  typedef std::unordered_map<std::string, ZipCache> zipcache_t;
479  typedef std::unordered_map<std::string, NewFile> new_files_t;
480 
481  File archive; //> File object for handling the ZIP archive
482  uint64_t archsize; //> size of the ZIP archive
483  bool cdexists; //> true if Central Directory exists, false otherwise
484  bool updated; //> true if the ZIP archive has been updated, false otherwise
485  std::unique_ptr<char[]> buffer; //> buffer for keeping the data to be parsed or raw data
486  std::unique_ptr<EOCD> eocd; //> End of Central Directory record
487  cdvec_t cdvec; //> vector of Central Directory File Headers
488  cdmap_t cdmap; //> mapping of file name to CDFH index
489  uint64_t cdoff; //> Central Directory offset
490  uint32_t orgcdsz; //> original CD size
491  uint32_t orgcdcnt; //> original number CDFH records
492  buffer_t orgcdbuf; //> buffer with the original CDFH records
493  std::unique_ptr<ZIP64_EOCD> zip64eocd; //> ZIP64 End of Central Directory record
494  OpenStages openstage; //> stage of opening / parsing a ZIP archive
495  std::string openfn; //> file name of opened file
496  zipcache_t zipcache; //> cache for inflating compressed data
497  std::unique_ptr<LFH> lfh; //> Local File Header record for the newly appended file
498  bool ckpinit; //> a flag indicating whether a checkpoint has been initialized
499  new_files_t newfiles; //> all newly appended files
500  };
501 
502 } /* namespace XrdZip */
503 
504 #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */
Write access is allowed.
Definition: XrdClXRootDResponses.hh:414
Definition: XrdClAnyObject.hh:32
static void Free(XRootDStatus *st, Response *rsp)
Free status and response.
Definition: XrdClZipArchive.hh:362
buffer_t orgcdbuf
Definition: XrdClZipArchive.hh:492
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:54
Definition: XrdClZipArchive.hh:444
cdvec_t cdvec
Definition: XrdClZipArchive.hh:487
NewFile(uint64_t offset, std::unique_ptr< LFH > lfh)
Definition: XrdClZipArchive.hh:458
ZipReadFromImpl< false > ReadFrom(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void *> buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:302
Directory list.
Definition: XrdClXRootDResponses.hh:649
Object stat info.
Definition: XrdClXRootDResponses.hh:399
Call the user callback.
Definition: XrdClResponseJob.hh:30
bool ckpinit
Definition: XrdClZipArchive.hh:498
void Clear()
Clear internal ZipArchive objects.
Definition: XrdClZipArchive.hh:428
Definition: XrdClZipArchive.hh:446
LFH of a newly appended file (in case it needs to be overwritten)
Definition: XrdClZipArchive.hh:456
StatInfo * make_stat(const std::string &fn)
Definition: XrdClZipArchive.hh:406
uint64_t offset
Definition: XrdClZipArchive.hh:470
Definition: XrdClZipArchive.hh:445
File archive
Definition: XrdClZipArchive.hh:481
XRootDStatus Stat(const std::string &fn, StatInfo *&info)
Definition: XrdClZipArchive.hh:199
std::unordered_map< std::string, ZipCache > zipcache_t
Type that maps file name to its cache.
Definition: XrdClZipArchive.hh:478
Definition: XrdEcStrmWriter.hh:52
XRootDStatus CloseFile()
Definition: XrdClZipArchive.hh:258
static XRootDStatus * make_status(const XRootDStatus &status=XRootDStatus())
Allocate new XRootDStatus object.
Definition: XrdClZipArchive.hh:420
AppendFileImpl< false > AppendFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint32_t > crc32, Arg< uint32_t > size, Arg< const void *> buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:421
void QueueJob(Job *job, void *arg=0)
Add a job to be run.
Definition: XrdClJobManager.hh:92
std::unordered_map< std::string, NewFile > new_files_t
Definition: XrdClZipArchive.hh:479
bool overwrt
Definition: XrdClZipArchive.hh:472
uint32_t GetFlags() const
Get flags.
cdmap_t cdmap
Definition: XrdClZipArchive.hh:488
A file.
Definition: XrdClFile.hh:45
XRootDStatus Write(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:155
std::unique_ptr< ZIP64_EOCD > zip64eocd
Definition: XrdClZipArchive.hh:493
Definition: XrdZipCDFH.hh:39
CloseArchiveImpl< false > CloseArchive(Ctx< ZipArchive > zip, uint16_t timeout=0)
Factory for creating CloseFileImpl objects.
Definition: XrdClZipOperations.hh:621
Definition: XrdClZipArchive.hh:449
zipcache_t zipcache
Definition: XrdClZipArchive.hh:496
none object for initializing empty Optional
Definition: XrdClOptional.hh:35
void Set(Type object, bool own=true)
Definition: XrdClAnyObject.hh:59
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errNotFound
Definition: XrdClStatus.hh:100
bool GetProperty(const std::string &name, std::string &value)
Get property on the underlying File object.
Definition: XrdClZipArchive.hh:293
Request status.
Definition: XrdClXRootDResponses.hh:218
Definition: XrdClAnyObject.hh:25
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
Definition: XrdClFileOperations.hh:358
static StatInfo * make_stat(const StatInfo &starch, uint64_t size)
Definition: XrdClZipArchive.hh:391
OpenArchiveImpl< false > OpenArchive(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags, uint16_t timeout=0)
Factory for creating OpenArchiveImpl objects.
Definition: XrdClZipOperations.hh:123
static void Schedule(ResponseHandler *handler, XRootDStatus *st, Response *rsp=nullptr)
Definition: XrdClZipArchive.hh:377
std::unique_ptr< EOCD > eocd
Definition: XrdClZipArchive.hh:486
std::unordered_map< std::string, size_t > cdmap_t
Definition: XrdZipCDFH.hh:54
uint32_t orgcdsz
Definition: XrdClZipArchive.hh:490
bool SetProperty(const std::string &name, const std::string &value)
Set property on the underlying File object.
Definition: XrdClZipArchive.hh:285
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:471
OpenStages
Stages of opening and parsing a ZIP archive.
Definition: XrdClZipArchive.hh:441
Handle an async response.
Definition: XrdClXRootDResponses.hh:1050
bool IsOpen()
Definition: XrdClZipArchive.hh:277
bool cdexists
Definition: XrdClZipArchive.hh:483
Definition: XrdEcReader.hh:57
uint32_t orgcdcnt
Definition: XrdClZipArchive.hh:491
void SetFlags(uint32_t flags)
Set flags.
Definition: XrdClZipArchive.hh:447
JobManager * GetJobManager()
Get the job manager object user by the post master.
XRootDStatus GetCRC32(const std::string &fn, uint32_t &cksum)
Definition: XrdClZipArchive.hh:232
XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:118
Definition: XrdClZipArchive.hh:45
Definition: XrdClZipArchive.hh:60
OpenFileImpl< false > OpenFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags=OpenFlags::None, Arg< uint64_t > size=0, Arg< uint32_t > crc32=0, uint16_t timeout=0)
Factory for creating OpenFileImpl objects.
Definition: XrdClZipOperations.hh:184
Nothing.
Definition: XrdClFileSystem.hh:77
NewFile(NewFile &&nf)
Definition: XrdClZipArchive.hh:464
std::vector< std::unique_ptr< CDFH > > cdvec_t
Definition: XrdZipCDFH.hh:44
std::string openfn
Definition: XrdClZipArchive.hh:495
OpenStages openstage
Definition: XrdClZipArchive.hh:494
new_files_t newfiles
Definition: XrdClZipArchive.hh:499
static PostMaster * GetPostMaster()
Get default post master.
Flags
Open flags, may be or&#39;d when appropriate.
Definition: XrdClFileSystem.hh:75
void SetSize(uint64_t size)
Set size.
XRootDStatus Stat(StatInfo *&info)
Definition: XrdClZipArchive.hh:218
Write operation (.
Definition: XrdClFileOperations.hh:367
Definition: XrdClZipArchive.hh:448
uint64_t cdoff
Definition: XrdClZipArchive.hh:489
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:497
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
std::unique_ptr< char[]> buffer
Definition: XrdClZipArchive.hh:485
uint64_t archsize
Definition: XrdClZipArchive.hh:482
Definition: XrdClZipArchive.hh:45
bool updated
Definition: XrdClZipArchive.hh:484
ZipListImpl< false > List(Ctx< ZipArchive > zip)
Factory for creating ZipStatImpl objects.
Definition: XrdClZipOperations.hh:574
static AnyObject * PkgRsp(Response *rsp)
Definition: XrdClZipArchive.hh:350