001 /*
002 Copyright (c) 1996-2012, Damon Hart-Davis
003 All rights reserved.
004
005 Redistribution and use in source and binary forms, with or without
006 modification, are permitted provided that the following conditions are
007 met:
008
009 * Redistributions of source code must retain the above copyright
010 notice, this list of conditions and the following disclaimer.
011
012 * Redistributions in binary form must reproduce the above copyright
013 notice, this list of conditions and the following disclaimer in the
014 documentation and/or other materials provided with the
015 distribution.
016
017 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
018 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
019 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
020 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
021 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
022 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
023 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
024 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
025 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
026 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
027 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028 */
029 package org.hd.d.pg2k.svrCore.datasource;
030
031 import java.io.BufferedInputStream;
032 import java.io.DataOutputStream;
033 import java.io.File;
034 import java.io.FileInputStream;
035 import java.io.FileNotFoundException;
036 import java.io.FileWriter;
037 import java.io.IOException;
038 import java.io.InputStream;
039 import java.io.InterruptedIOException;
040 import java.io.OutputStream;
041 import java.io.PrintWriter;
042 import java.io.RandomAccessFile;
043 import java.nio.ByteBuffer;
044 import java.security.DigestOutputStream;
045 import java.security.MessageDigest;
046 import java.text.SimpleDateFormat;
047 import java.util.ArrayList;
048 import java.util.BitSet;
049 import java.util.Calendar;
050 import java.util.Collections;
051 import java.util.Date;
052 import java.util.HashMap;
053 import java.util.HashSet;
054 import java.util.List;
055 import java.util.Map;
056 import java.util.Properties;
057 import java.util.Queue;
058 import java.util.Set;
059 import java.util.Vector;
060 import java.util.concurrent.Callable;
061 import java.util.concurrent.ConcurrentHashMap;
062 import java.util.concurrent.ExecutionException;
063 import java.util.concurrent.Future;
064 import java.util.concurrent.LinkedBlockingQueue;
065 import java.util.concurrent.atomic.AtomicBoolean;
066 import java.util.concurrent.atomic.AtomicReference;
067 import java.util.concurrent.locks.Lock;
068 import java.util.concurrent.locks.ReentrantLock;
069
070 import org.hd.d.pg2k.svrCore.AccessionData;
071 import org.hd.d.pg2k.svrCore.AllExhibitImmutableData;
072 import org.hd.d.pg2k.svrCore.AllExhibitProperties;
073 import org.hd.d.pg2k.svrCore.AllExhibitProperties.ExhibitDataSource;
074 import org.hd.d.pg2k.svrCore.CS8Bit;
075 import org.hd.d.pg2k.svrCore.CoreConsts;
076 import org.hd.d.pg2k.svrCore.ExhibitFile;
077 import org.hd.d.pg2k.svrCore.ExhibitName;
078 import org.hd.d.pg2k.svrCore.ExhibitPropsComputable;
079 import org.hd.d.pg2k.svrCore.ExhibitPropsGlobalImmutable;
080 import org.hd.d.pg2k.svrCore.ExhibitPropsLoadable;
081 import org.hd.d.pg2k.svrCore.ExhibitStaticAttr;
082 import org.hd.d.pg2k.svrCore.ExhibitThumbnails;
083 import org.hd.d.pg2k.svrCore.FileTools;
084 import org.hd.d.pg2k.svrCore.GenUtils;
085 import org.hd.d.pg2k.svrCore.MemoryTools;
086 import org.hd.d.pg2k.svrCore.Name;
087 import org.hd.d.pg2k.svrCore.Name.ExhibitFull;
088 import org.hd.d.pg2k.svrCore.ROByteArray;
089 import org.hd.d.pg2k.svrCore.Rnd;
090 import org.hd.d.pg2k.svrCore.SimpleLoggerIF;
091 import org.hd.d.pg2k.svrCore.Stratum;
092 import org.hd.d.pg2k.svrCore.TextUtils;
093 import org.hd.d.pg2k.svrCore.ThreadUtils;
094 import org.hd.d.pg2k.svrCore.Tuple;
095 import org.hd.d.pg2k.svrCore.MIME.ExhibitMIME;
096 import org.hd.d.pg2k.svrCore.props.GenProps;
097 import org.hd.d.pg2k.svrCore.props.LocalProps;
098 import org.hd.d.pg2k.svrCore.props.SecurityProps;
099 import org.hd.d.pg2k.svrCore.vars.BasicVarMgr;
100 import org.hd.d.pg2k.svrCore.vars.EventPeriod;
101 import org.hd.d.pg2k.svrCore.vars.EventVariableValue;
102 import org.hd.d.pg2k.svrCore.vars.SimpleVariableDefinition;
103 import org.hd.d.pg2k.svrCore.vars.SimpleVariableValue;
104 import org.hd.d.pg2k.svrCore.vars.SystemVariables;
105
106 import ORG.hd.d.IsDebug;
107
108 // FIXME: periodically/randomly (or upon demand) check for data file corruption.
109
110 /**Exhibit pipeline stage that fetches its data directly from a filesystem.
111 * This emulates the way that the pre-PG2K Gallery accesses its data.
112 * <p>
113 * That this considers itself to be the definitive master data store
114 * for persistent system variables (particularly event histories)
115 * and will try to use the persistent data area to store them,
116 * in a robust way with a history.
117 * <p>
118 * Past event values returned by this routine are considered to be
119 * authoritative, and upstream consumers can cache such values
120 * knowing them to be definitive.
121 * <p>
122 * All requests for valid events/periods should be responded to
123 * as authoritative or not depending on the slot time as follows:
124 * <ul>
125 * <li>In the future: null are returned because no data *can* exist yet.
126 * <li>For the current interval: a non-authoritative, non-null response.
127 * <li>For any past interval: an authoritative response.
128 * </ul>
129 * Where the data for responses does not yet exist,
130 * it must be invented with no events to indicate that
131 * it definitely does not exist
132 * so as to allow negative cacheing by upstream users.
133 * <p>
134 * This is assumed not to need to filter out duplicate event/var updates
135 * (nor bad timestamps) which is slow and may not even matter here.
136 * Any network connection upstream of us should do such filtering.
137 * <p>
138 * We don't aim to be massively efficient with this since we hope the
139 * cacheing stage that should normally be downstream of us will compensate
140 * for our main inefficiencies. We might, however, switch to some sort of
141 * non-blocking I/O in future so that failures of networked filesystems,
142 * (etc) internally won't stop Web service.
143 * <p>
144 * We locate configuration files and data using LocalProps.
145 */
146 public final class ExhibitDataFileSource implements SimpleExhibitPipelineIF
147 {
148 /**Create instance.
149 * Loads any persisted system variable values.
150 * @param logger logger to write to; null if no logging to be done
151 */
152 public ExhibitDataFileSource(final SimpleLoggerIF logger)
153 {
154 this.logger = (null != logger) ? logger : GenUtils.nullLogger;
155
156 varMgr = new BasicVarMgr(this.logger, true, true);
157
158 final File sysVarDir = _getEventHistoryStorageDir();
159 if(!sysVarDir.isDirectory())
160 { System.err.println("WARNING: event history storage dir not found: " + sysVarDir); }
161 else
162 {
163 // Don't block to load event histories if possible.
164 try { varMgr.loadEventHistories(sysVarDir, true); }
165 catch(final IOException e)
166 { System.err.println("WARNING: event history could not be restored from: " + sysVarDir); }
167 }
168 }
169
170 /**Logger; never null but may be a dummy. */
171 private final SimpleLoggerIF logger;
172
173 /**If true then when conserving power eliminate much activity that might force activity on the data area.
174 * This includes such things as checking status/flags,
175 * that might force an automount for example,
176 * using extra energy.
177 * <p>
178 * This may prove too conservative,
179 * eg refusing to look for new files at all even when explicitly indicated.
180 */
181 private static final boolean MINIMISE_FS_POWER = false;
182
183 /**The cached exhibit-data file to read; never null.
184 * This is relative to the data directory.
185 * <p>
186 * We don't construct the exhibit static
187 * data ourselves when called by any of the SimpleExhibitPipelineIF methods,
188 * but instead read a GZIPed serialised AllExhibitProperties object at
189 * the file given by this value, throwing an IOException if it cannot
190 * be read.
191 * <p>
192 * The createStaticCacheFile() routine can be used to create this
193 * file and do as much precomputation and preparation of thumbnails,
194 * etc, as possible off-line. Typically a command-line program
195 * would be used to run this each time the exhibits are changed,
196 * and the master will simply re-read the serialised file periodically.
197 * It is important in this case that the file be replaced atomically.
198 * The createStaticCacheFile() tries to create the file if it does not
199 * exist.
200 */
201 private static final File getStaticCacheFileName()
202 { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_CACHE_FILENAME)); }
203
204 /**The name of the AEP longHash file; never null.
205 * Typically, removing this file,
206 * or specifying an old/extant AEP hash different to this,
207 * will force the AEP to be reconstructed from the filesystem.
208 * <p>
209 * This is relative to the data directory.
210 * <p>
211 * Never touches the filesystem; simply constructs the name.
212 */
213 private static final File getStaticHashFileName()
214 { return(new File(LocalProps.getDataDir(), CoreConsts.FS_DATA_HASH_FILENAME)); }
215
216 /**Get the last/cached AEP longHash file content; null iff not present/readable.
217 * The elements are never null.
218 */
219 private static final Long getLastAEPHashIfAny()
220 {
221 final File staticHashFileName = getStaticHashFileName();
222 // If the hash file doesn't exist or is not readable or is zero length then ignore it.
223 if(!staticHashFileName.canRead() && (staticHashFileName.length() < 1)) { return(null); }
224
225 try
226 {
227 // Expect the file to consist of one line containing the decimal longHash.
228 final long hash = Long.parseLong(FileTools.readTextFile(staticHashFileName).trim(), 10);
229 return(hash);
230 }
231 // In case of error, log it, and return null.
232 catch(final Exception e) { e.printStackTrace(); return(null); }
233 }
234
235 /**Get the static attributes for a given exhibit; null if no such exhibit.
236 */
237 public ExhibitStaticAttr getStaticAttr(final ExhibitFull name)
238 throws IOException
239 {
240 // // If we have a static cache then get attrs via that.
241 // if(_getStaticCacheFileName() != null)
242 // { return(getAllExhibitProperties(-1L).aeid.getStaticAttr(name)); }
243 // // Else fetch directly from the filesystem.
244 return(_getStaticAttr(LocalProps.getDataDir(), name));
245 }
246
247 /**Get the static attributes for a given exhibit; null if the named exhibit does not exist.
248 * <p>
249 * This computes a new uncached value on each call.
250 * <p>
251 * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
252 */
253 private static ExhibitStaticAttr _getStaticAttr(final String fileRoot,
254 final Name.ExhibitFull name)
255 throws IOException
256 {
257 if((fileRoot == null) || (name == null))
258 { throw new IllegalArgumentException(); }
259
260 // With Name.ExhibitFull we know a priori that the value is 'safe', eg no ".." path components.
261 final String nameAsString = name.toString();
262 assert(!nameAsString.startsWith(".") && (nameAsString.indexOf("/.") == -1)) : "Dangerous name got through";
263
264 final File file = new File(fileRoot, nameAsString);
265 logFSAccess("reading static attrs for: "+file, false);
266 if(!file.canRead())
267 { return(null); }
268
269 return(new ExhibitStaticAttr(name,
270 file.length(),
271 file.lastModified()));
272 }
273
274 /**Read a chunk of the raw exhibit binary into the supplied buffer.
275 * The start position and an implied maximum read length are supplied.
276 * The start must be non-negative and no larger than the exhibit data length.
277 * <p>
278 * The call may return less than the the buffer capacity,
279 * though will block until it has read at least one byte unless at EOF or for a zero-byte request;
280 * this will be clear from the state of the buffer.
281 * <p>
282 * If a zero-byte request is made then the file may not actually be accessed.
283 * <p>
284 * This goes directly to the filesystem for each call.
285 * <p>
286 * TODO: prevent this from blocking indefinitely, even with a filesystem hang.
287 */
288 public void getRawFile(final ByteBuffer buf, final Name.ExhibitFull exhibitName, final int position, final boolean dontCache)
289 throws IOException
290 {
291 if(null == exhibitName) { throw new IllegalArgumentException(); }
292
293 // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
294
295 // if(!ExhibitName.validNameSyntax(exhibitName))
296 // { throw new IOException("invalid name (syntax)"); }
297 //
298 // // Do some extra security checks of requested file's name.
299 // if(exhibitName.startsWith(".") || (exhibitName.indexOf("/.") != -1))
300 // { throw new IOException("invalid name (unsafe)"); }
301
302 // Use of Name.ExhibitFull means that we know a priori that the name is 'safe'.
303 final String nameAsString = exhibitName.toString();
304 final File file = new File(LocalProps.getDataDir(), nameAsString);
305 logFSAccess("reading raw exhibit data from: "+file+", position="+position+", buf.remaining()="+buf.remaining(), false);
306 if(!file.exists() || !file.canRead())
307 { throw new FileNotFoundException(nameAsString); }
308
309 // Do the read!
310 RandomAccessFile raf = null;
311 try
312 {
313 raf = new RandomAccessFile(file, "r");
314 // raf.seek(position);
315 // raf.readFully(buf, offset, len);
316 raf.getChannel().read(buf, position);
317 }
318 catch(final IOException e)
319 {
320 System.err.println("IOException reading exhibit bytes start="+position+" name="+exhibitName+": " + e.getMessage());
321 throw e; // Rethrow the error.
322 }
323 finally { if(raf != null) { raf.close(); } } // Release handle ASAP.
324 }
325
326
327 /**Gets all static exhibit data if its timestamp is not that specified.
328 * If the time specified is negative then the object will be returned
329 * unconditionally.
330 * <p>
331 * If no exhibits are currently installed a then default set with a zero
332 * timestamp is returned.
333 * <p>
334 * If the caller's copy appears to be up-to-date (eg the oldStamp
335 * matches that that we would have been returned) then null is returned.
336 *
337 * @throws InterruptedIOException if another expensive call is already in progress; retry later
338 */
339 public AllExhibitImmutableData getAllExhibitImmutableData(final long oldStamp)
340 throws IOException
341 {
342 // if(_getStaticCacheFileName() != null)
343 // {
344 // final AllExhibitImmutableData extant = getAllExhibitProperties(-1L).aeid;
345 // // If caller has an up-to-date copy, return null.
346 // if((oldStamp >= 0) && (oldStamp == extant.timestamp))
347 // { return(null); }
348 // return(extant);
349 // }
350
351 return(_getAllExhibitImmutableData(LocalProps.getDataDir(), oldStamp, true));
352 }
353
354 /**Number of least-significant bytes of AllExhibitImmutableData timestamp to use as hash of full collection.
355 * Using a value of 1 would imply keeping about quarter-second timestamp
356 * precision (which does not exist in UNIX systems, typically).
357 * Using a value of 2 would imply about 1 minute precision.
358 * Using a value of 3 would imply about 5 hour precision.
359 * <p>
360 * Given that the Gallery is usually updated at intervals of at most
361 * hours to days normally, a value of 2 or 3 is probably optimal.
362 * <p>
363 * Must lie in the range 0 to 8 inclusive.
364 */
365 private static final int TS_LSbytes_HASH = 2;
366
367 /**Gets all immutable exhibit data if its timestamp is not that specified.
368 * If the time specified is negative the object will be returned
369 * unconditionally.
370 * <p>
371 * This does not use any cache, and computes afresh on each call,
372 * and is thread-safe.
373 * <p>
374 * If no exhibits are currently installed then a default set with a zero
375 * timestamp is returned.
376 * <p>
377 * If the caller's copy appears to be up-to-date (eg the oldStamp
378 * matches that that we would have been returned) null is returned.
379 * <p>
380 * We make the timestamp we use for the whole collection be the
381 * most significant bits of the latest-stamped exhibit and the
382 * least significant bits of the collection timestamp XORed with a hash
383 * over all the (sorted) ExhibitStaticAttr entries. We don't have many
384 * least significant bits, so we run a small-ish risk of missing some
385 * changes. In fact, we are quite rough-and-ready about the hash too!
386 * <p>
387 * We know that many (typically 10) of the least significant bits of
388 * the timestamp would not carry information anyway, eg with a
389 * one-second granularity in the UNIX filesystem.
390 * <p>
391 * To avoid causing too much confusion with our somewhat-faked
392 * timestamp, we limit it to between 1 and the current time of day
393 * in the worst case, though because we ensure that the fake
394 * timestamp calculated is no later than the timestamp of the newest
395 * exhibit then unless there is lots of clock-skew between our host
396 * and the file server we should not see this latter clamp actually used.
397 * The latter limit, if needed, can make the system inefficient just after
398 * a new exhibit has been added since our fake timestamp may seem
399 * to change on every call.
400 *
401 * @param careful if true, magic numbers of exhibits are checked and
402 * other extra-careful checking is done; this should be the default
403 * usage
404 */
405 public static AllExhibitImmutableData _getAllExhibitImmutableData(final String fileRoot,
406 final long oldStamp,
407 final boolean careful)
408 throws IOException
409 {
410 if(fileRoot == null)
411 { throw new IllegalArgumentException(); }
412 final File baseDir = new File(fileRoot);
413
414 // // Veto attempts to do this concurrently.
415 // if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
416 // try
417 // {
418 // Load exhibit set and (efficiently) sort it.
419 // We access the underlying files in sorted order aiming to improve filesystem performance.
420 final List<Name.ExhibitFull> names = new ArrayList<Name.ExhibitFull>(ExhibitFile.getFilesystemBasedExhibitNames(
421 GenUtils.systemOutLogger, baseDir, careful));
422 Collections.sort(names); // One-off sort of the names more efficient than incremental.
423
424 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[Found "+(names.size())+" exhibits based at "+baseDir+".]"); }
425
426 // Now iterate over the set in order,
427 // finding the newest exhibit,
428 // and computing a hash on all the static attr data.
429 // We also construct our map from names to attr at this point.
430 final Set<ExhibitStaticAttr> esas = new HashSet<ExhibitStaticAttr>(names.size() * 2);
431
432 long newest = 1; // Newest exhibit's timestamp, but forced strictly positive.
433
434 final MessageDigest md = GenUtils.getStandardDigest();
435 final DataOutputStream dos =
436 new DataOutputStream(
437 new DigestOutputStream(
438 (new OutputStream(){ // Null output stream...
439 @Override
440 public final void write(final int b) { }
441 @Override
442 public final void write(final byte[] b, final int off, final int len) { }
443 }),
444 md));
445 for(final Name.ExhibitFull name : names)
446 {
447 final ExhibitStaticAttr esa = _getStaticAttr(fileRoot, name);
448 if(null == esa) { continue; }
449
450 // We need to collect this ESAs...
451 esas.add(esa);
452
453 // Take care of the timestamp...
454 if(esa.timestamp > newest) { newest = esa.timestamp; }
455
456 // Update the hash...
457 final ExhibitFull fn = esa.getExhibitFullName();
458 dos.writeInt(fn.length());
459 dos.write(fn.toByteArray());
460 dos.writeLong(esa.length);
461 dos.writeLong(esa.timestamp);
462 dos.writeByte(1); // Acts as a nearly-unique separator.
463 }
464
465 dos.flush();
466 final byte[] hash = md.digest();
467
468 // XOR lsbytes in `newest' until we run out of hash bytes...
469 // ...first subtracting enough from the real timestamp to
470 // make space for the hash
471 // (ie so the computed value is no newer than the newest exhibit)...
472 long fakeStamp = newest - (1 << (8 * TS_LSbytes_HASH));
473 // XOR in LS bytes...
474 for(int i = Math.min(hash.length, TS_LSbytes_HASH); --i >= 0; )
475 { fakeStamp ^= ((long) (hash[i] & 0xff)) << (8 * i); }
476
477 // Clamp the fake timestamp to within allowed limit values,
478 // though this should not actually be necessary.
479 // If there are no exhibits then force a zero timestamp.
480 final long synthStamp;
481 if(esas.size() == 0) { synthStamp = 0; }
482 else
483 {
484 synthStamp = Math.min(Math.min(fakeStamp, newest), System.currentTimeMillis());
485 assert(synthStamp != 0);
486 }
487
488 // See if the client actually needs the value returned...
489 if((oldStamp >= 0) && (oldStamp == synthStamp))
490 { return(null); } // No, client is up-to-date.
491
492 // Now make the result!
493 final AllExhibitImmutableData result = new AllExhibitImmutableData(esas, synthStamp);
494
495 // Warn loudly if duplicate short names were found...
496 final Name.ExhibitFull dups[] = result.getFullNamesWithDuplicateShortNames();
497 if(dups.length > 0)
498 {
499 for(int i = 0; i < dups.length; ++i)
500 { System.err.println("WARNING: duplicate file component in: " + dups[i]); }
501 }
502
503 // Done!
504 return(result);
505 // }
506 // finally { _slow_op_lock.unlock(); }
507 }
508
509 /**Hash (longHash) of last-loaded AEP and timestamp, or -1 if no AEP yet loaded.
510 * The elements are never null.
511 * <p>
512 * Is volatile to allow lockless (read) access.
513 * <p>
514 * Updated by _getAllExhibitProperties().
515 */
516 private volatile long _AEP_last_longHash = -1;
517
518 /**Time before which we should not attempt to reload the AEP.
519 * Is volatile to allow lockless (read) access.
520 * <p>
521 * Updated by _getAllExhibitProperties().
522 * <p>
523 * Initially zero to force first recomputation.
524 */
525 private volatile long _AEP_nextLoad;
526
527 /**Reentrant lock avoid concurrent slow operations being attempted; never null.
528 * Used by both the internal getAEP and getAEID routines.
529 * <p>
530 * Rather than block, concurrent attempts may be vetoed with IOException.
531 */
532 private final ReentrantLock _slow_op_lock = new ReentrantLock();
533
534 /**Gets set of all exhibit properties if its hash is not that specified.
535 * If the hash specified is negative
536 * then the AEP will be loaded and returned unconditionally
537 * but this is likely to be expensive;
538 * call with the longHash of any extant item if at all possible.
539 * <p>
540 * If no exhibits are currently installed
541 * then a default empty set with a zero timestamp is returned.
542 * <p>
543 * If the caller's copy appears to be up-to-date
544 * (ie the oldHash matches that that would have been returned)
545 * then null is returned.
546 * <p>
547 * Note that a full load from the filesystem is likely to be slow/expensive.
548 *
549 * @throws InterruptedIOException if another expensive call is already in progress
550 * or we cannot otherwise currently attempt to (re)load an AEP;
551 * retry later
552 */
553 public AllExhibitProperties getAllExhibitProperties(final long oldHash)
554 throws IOException
555 {
556 // Generally we want to be very careful when (re)loading the exhibit set
557 // but when (temporarily) short of power then this may be a little more reckless
558 // and hope to notice any subtle issues when the system is flush.
559 final boolean carefulLoad = !GenUtils.mustConservePower();
560
561 return(_getAllExhibitProperties(oldHash, carefulLoad, false));
562 }
563
564
565 /**Minimum time between polls of the filesystem (ms) when not conserving power. */
566 private static final int MIN_POLL_MS = 11*60*1000;
567
568 /**Minimum time between polls of the filesystem (ms) when seriously conserving power. */
569 private static final int MIN_POLL_CONSERVING_MS = 121*60*1000;
570
571 /**Gets set of all exhibit properties if its hash is not that specified.
572 * If the hash specified is negative
573 * then the AEP will be loaded and returned unconditionally
574 * but this is likely to be expensive;
575 * call with the longHash of any extant item if at all possible.
576 * <p>
577 * If no exhibits are currently installed
578 * then a default empty set with a zero timestamp is returned.
579 * <p>
580 * If the caller's copy appears to be up-to-date
581 * (eg the oldHash matches that that would have been returned)
582 * then null is returned.
583 * <p>
584 * This is more averse to checking/reloading the AEP if conserving power.
585 *
586 * @param careful if true then magic numbers of exhibits are checked and
587 * other extra-careful checking is done; this should be the default usage
588 * @param forceLoadFromFilesystem if true, ignore any cached (hash) data and
589 * always load data from the filesystem
590 *
591 * @throws InterruptedIOException if another expensive call is already in progress
592 * or we cannot otherwise currently attempt to (re)load an AEP;
593 * retry later
594 */
595 private AllExhibitProperties _getAllExhibitProperties(final long oldHash,
596 final boolean careful,
597 final boolean forceLoadFromFilesystem)
598 throws IOException
599 {
600 // Warn if the caller may be forcing an unnecessary expensive full reload from the filesystem.
601 if(IsDebug.isDebug && !forceLoadFromFilesystem && (oldHash <= 0)) { System.err.println("WARNING: ExhibitDataFileSource._getAllExhibitProperties(): expensive 0 (empty AEP) or negative oldHash request"); }
602
603 // When (temporarily) conserving power and NOT forcing a (re)load from the filesystem,
604 // then unless a negative oldHash has been specified
605 // (ie the caller already has a non-empty AEP to hand)
606 // and if the caller's existing hash is what we remember from our last AEP load
607 // then this routine doesn't recheck the filesystem at all
608 // unless enough time has elapsed since the last load attempt,
609 // so as to avoid even 'waking' (ie spinning/powering up) the filesystem.
610 // When conserving power long term or in an emergency the wait a while *after* 'next load' time,
611 // meaning also that when power is available is is possible to very quickly test again.
612 // (If not conserving power then prodding the filesystem to ensure up-to-date is fine.)
613 final long startTime = System.currentTimeMillis();
614 if(!forceLoadFromFilesystem && (oldHash >= 0) &&
615 (startTime < _AEP_nextLoad + ((GenUtils.mustConservePowerLongTerm() || GenUtils.mustConservePowerExtreme()) ? (MIN_POLL_CONSERVING_MS - MIN_POLL_MS) : 0)))
616 {
617 // As far as we know the caller is up-to-date so return null.
618 if(_AEP_last_longHash == oldHash) { return(null); }
619 }
620
621 // If seriously conserving power and the caller has a non-empty AEP
622 // (that they can presumably work with for the time being)
623 // then veto trying to load/construct an AEP from the filesystem.
624 if(MINIMISE_FS_POWER && !forceLoadFromFilesystem && (oldHash > 0) && GenUtils.mustConservePowerExtreme())
625 { throw new InterruptedIOException("conserving power; will not attempt to load/construct AEP while caller has non-empty AEP"); }
626
627 // Try for reload from the filesystem (if not already busy)...
628 // Wrap this up to reserve a decent chunk of memory
629 // and at least shut out other memory-intensive actions.
630 if(!_slow_op_lock.tryLock()) { throw new InterruptedIOException("already in progress: "+_slow_op_lock); }
631 try
632 {
633 final Tuple.Pair<Boolean, AllExhibitProperties> result = MemoryTools.runMemoryIntensiveOperation((new Callable<AllExhibitProperties>()
634 {
635 @Override public AllExhibitProperties call() throws Exception
636 { return(_recomputeAEP(oldHash, careful, forceLoadFromFilesystem)); }
637 }),
638 true, // Try very hard to allow this to run.
639 AEP_LOAD_ESTIMATED_MIN_BYTES_REQUIRED);
640 if(!result.first) { throw new InterruptedIOException("unable to load AEP from filesystem; possibly not enough memory"); }
641 return(result.second);
642 }
643 catch(final IOException e) { throw e; }
644 catch(final Exception e) { throw new IOException("unable to load AEP from filesystem", e); }
645 finally { _slow_op_lock.unlock(); }
646 }
647
648 /**Wild guess at some of AEP-load fixed memory overhead, 16MB+ here seems like a good start; strictly positive. */
649 private static final int AEP_LOAD_ESTIMATED_MIN_BYTES_REQUIRED = 1<<14;
650
651 /**Do the slow and memory-intensive construction of the AEP from the filesystem. */
652 private AllExhibitProperties _recomputeAEP(final long oldHash,
653 final boolean careful, final boolean forceLoadFromFilesystem)
654 throws IOException, InterruptedIOException
655 {
656 final long startTime = System.currentTimeMillis();
657
658 // Iff there is a hash file containing the caller's specified (non-negative) oldHash
659 // then we assume that the caller's AEP is up-to-date.
660 // (After adding exhibits the hash file should be removed to force a later reload.)
661 final File staticHashFileName = getStaticHashFileName();
662 logFSAccess("checking cached AEP hash file " + staticHashFileName, false);
663 final Long hash = getLastAEPHashIfAny();
664 if(hash != null)
665 {
666 if(!forceLoadFromFilesystem && (oldHash >= 0) && (hash.longValue() == oldHash))
667 {
668 _AEP_last_longHash = oldHash; // Note the apparent longHash for the current exhibits.
669
670 // Simplified method to postpone next poll if no update was found...
671 final long endTime = System.currentTimeMillis();
672 final long waitTime = 101 + Math.max(MIN_POLL_MS, (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S/4) * 1000);
673
674 // Postpone next check.
675 final long nextUpdateTime = endTime + waitTime;
676 _AEP_nextLoad = nextUpdateTime;
677
678 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): caller is apparently up-to-date (see "+staticHashFileName+") with hash: "+oldHash+"; next check not before "+(new Date(nextUpdateTime))+".]"); }
679 return(null);
680 }
681 }
682
683 // Attempt to recover cache of entire AEP to avoid recomputing some very expensive state.
684 AllExhibitProperties oldProps = null;
685 final File staticCacheFileName = getStaticCacheFileName();
686 if(staticCacheFileName.canRead())
687 {
688 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): attempting to load old cache file to recover expensive-to-compute data: " + staticCacheFileName +" @ "+(new Date())+".]");
689 try { oldProps = (AllExhibitProperties) FileTools.deserialiseFromFile(staticCacheFileName, true); }
690 // Log but absorb any error, and do without cached data...
691 catch(final Exception e) { logger.log("failed to load cache file from "+staticCacheFileName, e); }
692 }
693
694 // Try to load/construct the AEP...
695 try
696 {
697 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): starting AEP construction from filesystem @ "+(new Date())+".]");
698
699 // Turn back on any muted warnings.
700 turnBackOnMutedWarnings();
701
702 final File baseDir = new File(LocalProps.getDataDir());
703 logFSAccess("loading AEP from raw exhibit data in "+baseDir, false);
704
705 // Load the EPGI (should be quick: do it first).
706 final ExhibitPropsGlobalImmutable epgi = ExhibitPropsGlobalImmutable.loadFromDataDir(baseDir);
707 if(ORG.hd.d.IsDebug.isDebug) { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded EPGI: "+epgi+"]"); }
708
709 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing AllExhibitImmutableData... @ " + (new Date())+".]"); }
710 // First, get immutable core data unconditionally...
711 final AllExhibitImmutableData aeid =
712 _getAllExhibitImmutableData(baseDir.getPath(), -1, careful);
713
714 // Create accession data for new exhibits...
715 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): creating new accession files @ "+(new Date())+".]"); }
716 createAccessionFiles(aeid);
717
718 // Get sorted set of exhibit names.
719 // We hope that iterating over a sorted set will result in
720 // better filesystem performance than an unsorted set.
721 final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
722
723 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): recomputing loadedProperties over exhibit set of "+(sortedNames.size())+"... "+(new Date())+".]"); }
724
725 // Collect the loaded and computable properties.
726 // We do this in parallel to maximise I/O throughput
727 // (though this may in fact be quite CPU- and memory- intensive)
728 // and use a ConcurrentHashMap as our working store as it is thread-safe.
729 // We use the CPU-intensive thread pool
730 // and wait for all work to complete.
731 //
732 // We recover all EPC data that we can from the previous/cached AEP if extant
733 // to save *lots* of time and effort.
734 final Map<Name.ExhibitFull,ExhibitPropsLoadable> loadedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsLoadable>(aeid.size() * 2);
735 final Map<Name.ExhibitFull,ExhibitPropsComputable> computedProps = new ConcurrentHashMap<Name.ExhibitFull, ExhibitPropsComputable>(aeid.size() * 2);
736 final List<Future<?>> tasks = new ArrayList<Future<?>>(MemoryTools.isMemoryStressed() ? 16 : sortedNames.size());
737 final ExhibitDataSource ds = makeExhibitDataSource();
738 final AllExhibitProperties c = oldProps;
739 final boolean recoverFromCache = (null != c);
740 for(final Name.ExhibitFull name : sortedNames)
741 {
742 // Create the task...
743 final Runnable r = new Runnable(){
744 public final void run()
745 {
746 // We don't store null/EMPTY EPL/EPC values.
747 try
748 {
749 // Always reload EPL data
750 // else we will miss changes in description, etc.
751 final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
752 if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY))
753 { loadedProps.put(name, epl); }
754
755 // Recover expensive EPC data if possible...
756 ExhibitPropsComputable epc = null;
757 if(recoverFromCache)
758 { epc = c.getExhibitPropsComputable(name); }
759 if((null == epc) || epc.equals(ExhibitPropsComputable.EMPTY))
760 { epc = ExhibitPropsComputable.createExhibitPropsComputable(aeid.getStaticAttr(name), ds); }
761 if((epc != null) && !epc.equals(ExhibitPropsComputable.EMPTY))
762 { computedProps.put(name, epc); }
763 }
764 catch(final Exception e)
765 { throw new RuntimeException("Could not create epl/epc for: " + name, e); }
766 }
767 };
768
769 // If we're currently pressed for resources then run this task synchronously...
770 if(MemoryTools.isMemoryStressed() || !ThreadUtils.couldRunLowPriorityDiscardableTask())
771 { try { r.run(); } catch(final Exception e) { throw new IOException(e); } }
772 // ...else attempt to run the task concurrently for better throughput.
773 else
774 { tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(r)); }
775 }
776
777 // Wait for all asynchronous tasks to finish...
778 for(final Future<?> task : tasks)
779 {
780 try { task.get(); }
781 catch(final InterruptedException e)
782 {
783 final InterruptedIOException err = new InterruptedIOException(e.getMessage());
784 err.initCause(e);
785 throw err;
786 }
787 catch(final ExecutionException e)
788 {
789 final IOException err = new IOException(e.getMessage());
790 err.initCause(e);
791 throw err;
792 }
793 }
794
795 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): constructing AEP... @ "+(new Date())+"]"); }
796
797 // Construct a new AEP from the data gathered.
798 final AllExhibitProperties newProps = new AllExhibitProperties(null,
799 epgi,
800 aeid,
801 loadedProps,
802 computedProps,
803 0);
804
805 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): compacting... @ "+(new Date())+"]");
806 newProps.compact();
807
808 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): loaded and constructed new AEP with longHash="+newProps.longHash+" @ "+(new Date())+"]");
809
810 // Cache the longHash in memory...
811 _AEP_last_longHash = newProps.longHash;
812 // Save a hash file recording the AEP longHash (in decimal) if possible.
813 // May not be possible if the filesystem is read-only for example.
814 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing AEP hash to "+staticHashFileName+"... @ "+(new Date())+"]");
815 try { FileTools.replacePublishedFile(staticHashFileName.getPath(), (new CS8Bit(Long.toString(newProps.longHash, 10))).toByteArray()); }
816 catch(final IOException e) { e.printStackTrace(); }
817
818 // If the AEP has changed from the static cached version (if any) then try to save it for next time.
819 if((oldProps == null) || (oldProps.longHash != newProps.longHash))
820 {
821 System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): cacheing serialised AEP to "+staticCacheFileName+"... @ "+(new Date())+"]");
822 try { FileTools.serialiseToFile(newProps, staticCacheFileName, true, !IsDebug.isDebug); }
823 // Log but absorb any errors.
824 catch(final Exception e) { e.printStackTrace(); }
825 }
826
827 // If the caller really is up-to-date then return null (discarding the new AEP).
828 if((oldHash >= 0) && (newProps.longHash == oldHash)) { return(null); }
829
830 // Return the newly-loaded AEP.
831 return(newProps);
832 }
833 finally
834 {
835 // Whether we succeeded or not...
836 // work out the minimum wait before starting the next load.
837 //
838 // Usually, after a successful load,
839 // the next load attempt should not happen within the normal slackness interval.
840 //
841 // This tries not to spend more than ~10% of real time doing loads if possible,
842 // but caps the *minimum* wait before another attempt to a few hours.
843 // We throw in a small random factor too, to help avoid collisions...
844 //
845 // The minimum polling interval is boosted post-hoc (not here) when conserving power.
846 final long endTime = System.currentTimeMillis();
847 final long timeTaken = endTime - startTime;
848 final long minWaitTime = Math.max(MIN_POLL_MS, CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * (1000 / 10));
849 final long waitTime = Math.max(minWaitTime, // Default/minimum wait time...
850 Math.min(timeTaken << 3, // Limit effort to ~10% of wall-clock time if possible.
851 2 * 3600 * 1000L) + // Limit delay to next attempt to ~2 hours at worst.
852 Rnd.fastRnd.nextInt(10001 + (CoreConsts.DEFAULT_TEMPORAL_SLACKNESS_S * 101)));
853
854 // Schedule next check.
855 final long nextUpdateTime = endTime + waitTime;
856 _AEP_nextLoad = nextUpdateTime;
857 /* if(ORG.hd.d.IsDebug.isDebug) */ { System.out.println("[ExhibitDataFileSource._getAllExhibitProperties(): done new AEP load: next check not before "+(waitTime/(1000 * 60))+"mins, at "+(new Date(nextUpdateTime))+".]"); }
858 }
859 }
860
861 /**Wrap this instance as an ExhibitDataSource.
862 * Only really meant for internal use,
863 * but may be useful for low-level use of the file-based data.
864 */
865 public AllExhibitProperties.ExhibitDataSource makeExhibitDataSource()
866 {
867 return(new AllExhibitProperties.ExhibitDataSource(){
868 @Override
869 public final void getRawFile(final ByteBuffer buf, final ExhibitFull exhibitName, final int position)
870 throws IOException
871 { ExhibitDataFileSource.this.getRawFile(buf, exhibitName, position, false); }
872 @Override
873 public final boolean isExhibitFullyLoaded(final ExhibitStaticAttr esa)
874 { return(true); }
875 });
876 }
877
878
879 /**Gets the general properties as a GenProps object if its timestamp is not that specified.
880 * If the time specified is negative the object will be returned unconditionally.
881 * <p>
882 * If no props are currently installed/available a default set with a zero
883 * timestamp is returned.
884 * <p>
885 * If the caller's copy appears to be up-to-date (eg the oldStamp
886 * matches that that we would have been returned) null is returned.
887 * <p>
888 * This computes a new uncached value on each call.
889 */
890 public GenProps getGenProps(final long oldStamp)
891 throws IOException
892 {
893 final File file = new File(LocalProps.getConfDir(), CoreConsts.FS_CONF_SYSPROPS);
894
895 // If the stamp presented is < 0 or different to that of the file,
896 // reread the file now.
897 final long newStamp = file.lastModified();
898 if((oldStamp < 0) || (oldStamp != newStamp))
899 {
900 try
901 {
902 final FileInputStream fis = new FileInputStream(file);
903 try
904 {
905 final Properties props = FileTools.loadProperties(new BufferedInputStream(fis));
906 return(new GenProps(props, newStamp));
907 }
908 finally { fis.close(); }
909 }
910 catch(final IOException e)
911 {
912 logger.log("ExhibitDataFileSource: I/O problem loading/parsing GenProps", e);
913 throw e; // Pass the error on.
914 }
915 catch(final Exception e)
916 {
917 final String message = "ExhibitDataFileSource: problem loading/parsing GenProps";
918 logger.log(message, e);
919 throw new IOException(message, e); // Pass the error on.
920 }
921 }
922
923 return(null); // Caller's copy seems to be up-to-date.
924 }
925
926 /**Gets the generic security properties as a Properties object if its timestamp is not that specified.
927 * If the time specified is negative the object will be returned unconditionally.
928 * <p>
929 * If no props are currently installed/available a default set with a zero
930 * timestamp is returned.
931 * <p>
932 * If the caller's copy appears to be up-to-date (eg the oldStamp
933 * matches that that would have been returned) null is returned.
934 * <p>
935 * This computes a new uncached value on each call.
936 * <p>
937 * These generic properties are fetchable over the network, for example,
938 * and need not be present locally at each host the JVM runs on.
939 */
940 public java.util.Properties getGenSecProps(final long oldStamp)
941 throws IOException
942 {
943 final SecurityProps sp =
944 SecurityProps.getSecurityPropsUncachedFromFilesystem(oldStamp);
945
946 // If security props file has not changed, return null.
947 if(sp == null)
948 { return(null); }
949
950 // Extract the relatively-safe "generic" component.
951 return(sp.getGenSecProps());
952 }
953
954 /**Suffix appended to serialised thumbnail class data for each exhibit.
955 * Such files are beside the original exhibit file,
956 * are prefixed with FileTools.F_permPrefix,
957 * and suffixed with this (which is chosen so as not to look like an exhibit).
958 * <p>
959 * The file is a GZIPped, serialised, ExhibitThumbnails class if present.
960 */
961 public static final String THUMBNAIL_SUFFIX = ".thumbnails";
962
963 /**If true, the serialised thumbnail files are compressed (GZIPped) on disc.
964 * This might cost a little time but might possibly save a little space.
965 * Note that thumbnails may in any case be GZIPped in transit to a slave server.
966 */
967 public static final boolean THUMBNAILS_ARE_GZIPPED = false;
968
969 /**If true, we have warned about thumbnail directory absence.
970 * Starts, false; may be cleared periodically.
971 * Set true when we have warned.
972 * <p>
973 * Volatile to allow read/write without a lock.
974 * <p>
975 * Not critical to correct operation,
976 * so we can tolerate races and sloppy handling.
977 */
978 private volatile boolean _haveWarnedNoTNDir;
979
980 /**Turn back on warnings we have muted.
981 * We do this occasionally so that persistent faults
982 * will show up in the log and don't just vanish.
983 * <p>
984 * Avoids locking unless really necessary.
985 */
986 private void turnBackOnMutedWarnings()
987 {
988 _haveWarnedNoTNDir = false;
989 }
990
991
992 /**Interval between logging filesystem accesses (ms); strictly positive.
993 * Should be short enough to reveal operations, possibly unwanted,
994 * that 'wake up' the filesystem (eg cause it to power-up, get mounted, etc).
995 * <p>
996 * Should be long enough to eliminate most uninteresting stuff
997 * such as the multiple accesses to load/examine one file.
998 * Keeping the interval to under a minute will most likely catch most 'wake-ups'.
999 */
1000 private static final int FS_LOG_INTERVAL_MS = 59000; // Just less than a minute.
1001
1002 /**Count of filesystem access operations logged including those not displayed; never null.
1003 * Accessed under _FSLogLock.
1004 */
1005 private static int countFSAccess;
1006
1007 /**Time that last filesystem access was logged; initially zero.
1008 * Accessed under _FSLogLock.
1009 */
1010 private static long lastLoggedFSAccess;
1011
1012 /**Lock for logging filesystem access; never null. */
1013 private static final ReentrantLock _FSLogLock = new ReentrantLock();
1014
1015 /**Log access to data filesystem.
1016 * Useful for tracing unwanted activity.
1017 * <p>
1018 * May limit output (eg to once every few minutes)
1019 * to still give traceability of 'wakeup' operations.
1020 *
1021 * @param force if true then force printing
1022 */
1023 private static void logFSAccess(final String detail, final boolean force)
1024 {
1025 _FSLogLock.lock();
1026 try
1027 {
1028 final int count = ++countFSAccess;
1029
1030 // Discard this log message if too soon since the previous...
1031 final long now = System.currentTimeMillis();
1032 if(!force && (now < lastLoggedFSAccess + FS_LOG_INTERVAL_MS)) { return; }
1033 lastLoggedFSAccess = now;
1034
1035 // Write this long message.
1036 System.out.println("ExhibitDataFileSource: DATA FILESYSTEM ACCESS "+count+": "+detail+" @ "+(new Date(now)));
1037 }
1038 finally { _FSLogLock.unlock(); }
1039 }
1040
1041
1042 /**Gets the thumbnails for an exhibit; null if not (currently) available.
1043 * In this implementation we never try to create thumbnails
1044 * (we assume that the source directory is read only for example),
1045 * but we'll return one if we can find one.
1046 * <p>
1047 * Thumbnails are looked for by default in the exhibits directory
1048 * tree beside the source exhibits, but a LocalProps value
1049 * can override this so that source data can be separated
1050 * from derived data.
1051 *
1052 * @param create if true then may try to create and cache missing thumbnails,
1053 * but creating the thumbnail is only attempted
1054 * if cacheing is likely to be successful
1055 * because this should be a once-only operation per exhibit
1056 */
1057 public ExhibitThumbnails getThumbnails(final ExhibitFull name, final boolean create)
1058 throws IOException
1059 {
1060 logFSAccess("requesting pre-computed thumbnails for: "+name, false);
1061
1062 // In the simple case where we know that thumbnails
1063 // definitely can't be created (from the type)
1064 // return the NO_THUMBNAILS value immediately,
1065 // ignoring the create parameter.
1066 // We never cache this (negative) value.
1067 final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(name));
1068 if((type == null) || (type.handler == null) ||
1069 !type.canPossiblyCreateThumbnailOfSameMIMEType())
1070 { return(ExhibitThumbnails.NO_THUMBNAILS); }
1071
1072 final String dataDir = LocalProps.getDataDir();
1073 final ExhibitStaticAttr esa = _getStaticAttr(dataDir, name); // Avoid tickling cache badly...
1074 // If the exhibit appears not to exist
1075 // then return null rather than NO_THUMBNAILS
1076 // (for example the exhibit may be in the process or being added).
1077 if(esa == null)
1078 { return(null); }
1079
1080 // Construct the name of the thumbnails' file if it exists.
1081 final String relPath = thumbnailRelPath(name);
1082
1083 // Compute full path from exhibit directory.
1084 final File topDir = new File(dataDir, LocalProps.getThumbnailRelDir());
1085
1086 // If the top directory for the thumbnails does not exist,
1087 // then we definitely will not be returning thumbnails.
1088 // We also warn when this directory does not exist.
1089 final boolean noTNDir = !topDir.isDirectory();
1090 if(noTNDir)
1091 {
1092 if(!_haveWarnedNoTNDir)
1093 {
1094 System.err.println("ExhibitDataFileSource: WARNING: top-level thumbnail directory absent: " + topDir);
1095 _haveWarnedNoTNDir = true;
1096 }
1097
1098 // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1099 // since thumbnails may be available/generated later.
1100 return(null);
1101 }
1102
1103 // If the thumbnail file seems readable, etc, then try to load it.
1104 // Any problem, eg an IOException, will be propagated to the caller.
1105 final File fullPath = new File(topDir, relPath);
1106 logFSAccess("reading thumbnail from: "+fullPath, false);
1107 if(fullPath.exists() && fullPath.canRead() && fullPath.isFile())
1108 {
1109 final ExhibitThumbnails result = (ExhibitThumbnails)
1110 FileTools.deserialiseFromFile(fullPath, THUMBNAILS_ARE_GZIPPED);
1111 logFSAccess("read thumbnail from: "+fullPath+": "+result, ExhibitThumbnails.NO_THUMBNAILS.equals(result)); // Force result for NO_THUMBNAILS value...
1112 return(result);
1113 }
1114
1115 // Only even consider thumbnail creation
1116 // if the thumbnail area appears to be writable.
1117 if(create && topDir.canWrite())
1118 {
1119 // TODO
1120 }
1121
1122 // Can't seem to find a thumbnail file,
1123 // so tell the caller no thumbnails at the moment.
1124 // We don't return ExhibitThumbnails.NO_THUMBNAILS,
1125 // since thumbnails may be available/generated later.
1126 return(null);
1127 }
1128
1129 /**Poll periodically.
1130 * Periodically save the system variables.
1131 */
1132 public void poll(final GenProps gp)
1133 {
1134 _saveSystemVariables(varMgr, false);
1135 }
1136
1137 /**Save the system variables (and logs).
1138 * Mainly save the histories of persistent events.
1139 * <p>
1140 * We save these with a history so that in effect we have backups
1141 * if something bad happens
1142 * (though this implies that some other housekeeping will
1143 * have to clear up excess copies, etc,
1144 * in order to avoid space requirements growing without bound).
1145 * <p>
1146 * The data is saved in the persistent data area.
1147 * <p>
1148 * Reports but does not propagate exceptions.
1149 *
1150 * @param vars set of variables to save
1151 * @param force if true, force an immediate complete save
1152 */
1153 private void _saveSystemVariables(final BasicVarMgr vars,
1154 final boolean force)
1155 {
1156 // Return immediately if too soon to save again
1157 // (postponed further when conserving energy)
1158 // and a save is not being forced.
1159 if(!force &&
1160 (_nextVarSaveEarliest + (GenUtils.mustConservePower() ?
1161 Math.max(CoreConsts.ASYNC_MIN_POWER_SAVE_NON_CRITICAL_DATA_FLUSH_MS, _VAR_SAVE_INTERVAL_MS*3) : 0) >
1162 System.currentTimeMillis()))
1163 { return; }
1164
1165 // Keeps a history and attempts just to save changes...
1166 try
1167 {
1168 // First write out the persistent events as-is which should free some memory.
1169 _appendToEventLogs();
1170
1171 // Unless doing a forced save, eg at shut-down,
1172 // do an incremental changes-only save to be as quick as reasonably possible.
1173 // When forced, push out *all* stores, not just changed ones,
1174 // to minimise the chance of entirely 'losing' very-slowly-changing event sets.
1175 vars.saveEventHistories(_getEventHistoryStorageDir(), !force, true, !force);
1176 // The save seems to have completed fine, so postpone the next one.
1177 _nextVarSaveEarliest = System.currentTimeMillis() + _VAR_SAVE_INTERVAL_MS +
1178 Rnd.fastRnd.nextInt(1 + (_VAR_SAVE_INTERVAL_MS/4));
1179 }
1180 catch(final IOException e)
1181 { logger.log("problem persisting vars to filesystem", e); }
1182 }
1183
1184
1185
1186 /**Our private static instance of a GMT calendar.
1187 * No one must change the ZONE or DST offset.
1188 */
1189 private static final Calendar GMTCalendar = Calendar.getInstance();
1190 /**Initialise GMTCalendar. */
1191 static
1192 {
1193 // Set to GMT
1194 // i.e 0 Offset
1195 GMTCalendar.set(Calendar.ZONE_OFFSET, 0);
1196 // and 0 Daylight savings
1197 GMTCalendar.set(Calendar.DST_OFFSET, 0);
1198 }
1199
1200 /**Format we use to insert into event history file name. */
1201 private static final SimpleDateFormat dateFmtEHFile =
1202 new SimpleDateFormat("yyyyMMdd");
1203 /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1204 static { dateFmtEHFile.setCalendar(GMTCalendar); }
1205
1206 /**Format we use for timestamp to insert into event log file name. */
1207 private static final SimpleDateFormat timestampFmtEHFile =
1208 new SimpleDateFormat("yyyyMMdd-HHmmss.SSS");
1209 /**Set the Calendar for dateFmtEHFile to be the GMT calendar. */
1210 static { timestampFmtEHFile.setCalendar(GMTCalendar); }
1211
1212 /**Max log line length for event log. */
1213 private static final int MAX_EV_LOG_LINE_LENGTH = 4095;
1214
1215 /**Private lock for _appendToEventLogs(), static to serialise all filesystem access. */
1216 private static final Lock _aTWL_lock = new ReentrantLock();
1217
1218 /**Write out the accumulated event values to disc.
1219 * These are written to date-stamped files to allow automatic "rolling".
1220 * <p>
1221 * We keep a separate file for each named event,
1222 * and append events to each appropriate file in order
1223 * with a record of the form:
1224 * <pre>
1225 * YYYYMMDD-HHmmss.SSS eventName eventValue
1226 * </pre>
1227 * with a null value being indicated by the value "null".
1228 * <p>
1229 * Note that the timestamp is that of the value,
1230 * not the time that the event is received or written.
1231 * <p>
1232 * We may cache file handles to reduce the cost of opening and closing
1233 * files for each event,
1234 * but in any case will have closed all log files opened by us
1235 * by the time this routine returns.
1236 * <p>
1237 * We synchronise on a private static lock to try to prevent two instances of this
1238 * running concurrently and leading to file corruption or misordering the logs;
1239 * a second concurrent attempt is silently vetoed.
1240 */
1241 private void _appendToEventLogs()
1242 throws IOException
1243 {
1244 // If nothing to write then return immediately.
1245 if(eventsToLog.isEmpty()) { return; }
1246
1247 // If already running then return immediately.
1248 if(!_aTWL_lock.tryLock()) { return; }
1249 try
1250 {
1251 // Attempt to write out accumulated event values to the logs.
1252 final List<SimpleVariableValue> events;
1253 // Move all events into working copy atomically to hold lock for short time.
1254 synchronized(eventsToLog) { events = new ArrayList<SimpleVariableValue>(eventsToLog); eventsToLog.clear(); }
1255
1256 //System.out.println("[EVENTS TO LOG: ~" + size + "...]");
1257
1258 // Separate the events into groups (preserving order) by event name
1259 // so that we can do one file operation for all instances of one event.
1260 final Map<String, List<SimpleVariableValue>> groupedEvents = new HashMap<String, List<SimpleVariableValue>>();
1261 for(final SimpleVariableValue svv : events)
1262 {
1263 final String name = svv.getDef().getName();
1264 List<SimpleVariableValue> evvs = groupedEvents.get(name);
1265 if(evvs == null)
1266 {
1267 evvs = new ArrayList<SimpleVariableValue>();
1268 groupedEvents.put(name, evvs);
1269 }
1270 evvs.add(svv);
1271 }
1272
1273 // Check that the log directory is writable.
1274 // Complain loudly and discard the events if not.
1275 final File logDir = _getEventLogDir();
1276 if((logDir == null) || !logDir.isDirectory() || !logDir.canWrite())
1277 {
1278 System.err.println("WARNING: persistent event log dir "+logDir+" not usable: discarding "+events.size()+" event(s).");
1279 return;
1280 }
1281
1282 // Create/format the date string once, if we are going to use it.
1283 final String dateString = "." + dateFmtEHFile.format(new Date());
1284
1285 // Any error accumulated so far...
1286 IOException err = null;
1287
1288 for(final String name : groupedEvents.keySet())
1289 {
1290 final StringBuilder sb = new StringBuilder(80);
1291 sb.append("eventLog.");
1292 sb.append(name);
1293 sb.append(BasicVarMgr.EVENT_STORE_NAMETERM);
1294 sb.append(dateString);
1295 sb.append(".log");
1296 final File f = new File(logDir, sb.toString());
1297
1298 try
1299 {
1300 final PrintWriter pw = new PrintWriter(new FileWriter(f, true)); // Append to existing file.
1301 try
1302 {
1303 for(final SimpleVariableValue svv : groupedEvents.get(name))
1304 {
1305 final StringBuilder logLine = new StringBuilder(80);
1306
1307 logLine.append(timestampFmtEHFile.format(new Date(svv.getTimestamp())));
1308 logLine.append(' ');
1309 logLine.append(name);
1310 logLine.append(' ');
1311 logLine.append(svv.getValue());
1312
1313 // Truncate VERY long entries...
1314 if(logLine.length() > MAX_EV_LOG_LINE_LENGTH)
1315 {
1316 logLine.setLength(MAX_EV_LOG_LINE_LENGTH - 3);
1317 logLine.append("..."); // Indicate truncation.
1318 }
1319
1320 // We sanitise the log line to ensure that it
1321 // contains no control codes especially CRs or LFs
1322 // that would disrupt the record-per-line format.
1323 // Replace such toxic chars with something innocuous.
1324 for(int i = logLine.length(); --i >= 0; )
1325 {
1326 final char ch = logLine.charAt(i);
1327 if((ch < 32) || (ch > 255) || (ch == 127))
1328 { logLine.setCharAt(i, ' '); }
1329 }
1330
1331 // Write entry in own line to EOF...
1332 pw.println(logLine.toString());
1333 }
1334
1335 // Flush to disc...
1336 pw.flush();
1337 }
1338 finally { pw.close(); }
1339 }
1340 catch(final IOException e)
1341 {
1342 e.printStackTrace();
1343 err = e; // Absorb error for now. Rethrow later.
1344 }
1345 }
1346
1347 // If we encountered any error then (re)throw a sample exception here.
1348 if(err != null) { throw err; }
1349 }
1350 finally { _aTWL_lock.unlock(); }
1351 }
1352
1353 /**Minimum interval between saves, ms; strictly positive.
1354 * Intended to avoid losing too much data at shut-down,
1355 * yet avoid wearing out the disc (or other storage such as Flash) and consuming CPU time
1356 * with redundant work.
1357 * <p>
1358 * Something from tens of seconds to a few minutes is probably about right.
1359 */
1360 private static final int _VAR_SAVE_INTERVAL_MS = 5*60*1000;
1361
1362 /**Earliest time after which to allow the next event save (zero if no save yet completed).
1363 * Marked volatile to allow thread-safe lock-free access.
1364 */
1365 private transient volatile long _nextVarSaveEarliest;
1366
1367
1368 /**Get directory used for storing system-variable persistent event histories; never null.
1369 * Though never null, the indicated directory may not exist
1370 * or may otherwise not be usable.
1371 * <p>
1372 * We derive this from the LocalProps value.
1373 */
1374 private static File _getEventHistoryStorageDir()
1375 {
1376 return(new File(LocalProps.getPersistentStateDir(), "history"));
1377 }
1378
1379 /**Get directory used for storing system-variable persistent event logs; never null.
1380 * Though never null, the indicated directory may not exist
1381 * or may otherwise not be usable.
1382 * <p>
1383 * We get derive this from the LocalProps value.
1384 */
1385 private static File _getEventLogDir()
1386 {
1387 return(new File(LocalProps.getPersistentStateDir(), "log"));
1388 }
1389
1390 /**Our set of system variables.
1391 * Persistent values are loaded from disc on first attempt to
1392 * get and variable(s) if possible.
1393 * <p>
1394 * We expect this to be at the top of the data pipe on the master server,
1395 * and thus handle all the system non-local variables,
1396 * plus all the local variables of the master.
1397 * <p>
1398 * If any persistent values are set then they all get written to disc
1399 * as compressed serialised data on the next poll().
1400 * (Doing saves on each poll() helps reduce expensive disc writes by
1401 * grouping multiple updates into one save if they are happening rapidly.)
1402 * <p>
1403 * The current set or variables (including locals) is held in memory,
1404 * which also allows us to merge globals if required.
1405 * <p>
1406 * The varMgr is marked as an end-point so that it will
1407 * generate and return a unique local system ID.
1408 * <p>
1409 * This is assumed not to need to filter out duplicate updates
1410 * (nor bad timestamps) which is slow and may not even matter here.
1411 * Any network connection upstream of us should do such filtering.
1412 * <p>
1413 * We have the varMgr screen out repeats and bad timestamps, etc.
1414 */
1415 private final BasicVarMgr varMgr; // Endpoint.
1416
1417 /**In-order List/Queue thread-safe list of persistent events to be logged; never null.
1418 * Remove items FIFO or lock the queue and remove all the items in one go.
1419 * <p>
1420 * We don't synchronously save these data.
1421 */
1422 private final Vector<SimpleVariableValue> eventsToLog = new Vector<SimpleVariableValue>();
1423
1424 /**Set variable value; persistent values will eventually go to disc.
1425 *
1426 * @param newValue
1427 * @throws IOException
1428 */
1429 public void setVariable(final SimpleVariableValue newValue)
1430 throws IOException
1431 {
1432 varMgr.setVariable(newValue);
1433
1434 // Capture any persistent event for later logging.
1435 if(newValue != null)
1436 {
1437 final SimpleVariableDefinition def = newValue.getDef();
1438 if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1439 { queueNewEventToLog(newValue); }
1440 }
1441 }
1442
1443 /**Maximum number of log entries to hold in memory before flushing; strictly positive.
1444 * Should be small enough not to impact memory too much
1445 * (including the entailed effects of intern()ed/singleton values and so on)
1446 * but large enough that generally the periodic poll()-driven flush
1447 * will take care of things, keeping setVariable() and setVariables() fast.
1448 * <p>
1449 * Made public to help other pipeline cacheing stages set related limits.
1450 */
1451 public static final int MAX_LOG_ENTRIES_QUEUED = 1024;
1452
1453 /**Queue one new event to be logged.
1454 * If the number of accrued events becomes too large
1455 * then flush to the filesystem synchronously and liberate some memory.
1456 * (Generally we hope that background polling will flush logs as needed.)
1457 */
1458 private void queueNewEventToLog(final SimpleVariableValue newValue)
1459 throws IOException
1460 {
1461 eventsToLog.add(newValue);
1462
1463 // Flush the queue synchronously if getting too large.
1464 // Flush sooner if the system is short of memory.
1465 if(eventsToLog.size() > (MemoryTools.isMemoryStressed() ? (MAX_LOG_ENTRIES_QUEUED>>2) : MAX_LOG_ENTRIES_QUEUED))
1466 { _appendToEventLogs(); }
1467 }
1468
1469 /**Set variable values; persistent values will eventually go to disc.
1470 *
1471 * @param newValues
1472 * @throws IOException
1473 */
1474 public int setVariables(final SimpleVariableValue[] newValues)
1475 throws IOException
1476 {
1477 final int n = varMgr.setVariables(newValues);
1478
1479 // Capture any persistent events for later logging, in order.
1480 for(final SimpleVariableValue svv : newValues)
1481 {
1482 if(svv != null)
1483 {
1484 final SimpleVariableDefinition def = svv.getDef();
1485 if(def.isEvent() && def.isPersistent() && SystemVariables.defs.contains(def))
1486 { queueNewEventToLog(svv); }
1487 }
1488 }
1489
1490 return(n);
1491 }
1492
1493 /**Get variable value; persistent values may have come from disc.
1494 */
1495 public SimpleVariableValue getVariable(final SimpleVariableDefinition var)
1496 {
1497 return(varMgr.getVariable(var));
1498 }
1499
1500 /**Get variable values; persistent values may have come from disc.
1501 */
1502 public SimpleVariableValue[] getVariables(final long changedSince)
1503 {
1504 return(varMgr.getVariables(changedSince));
1505 }
1506
1507 /**Synchronise variable values.
1508 * This implementation <em>does not</em> force a flush to disc.
1509 */
1510 public void syncVariables(final boolean force)
1511 {
1512 // Nothing needed in this implementation.
1513 // In particular this does not flush values to disc.
1514 }
1515
1516 /**Get the current partial, or previous full, event set at the specified interval; never null.
1517 * This is a simplified interface to return either the current event set
1518 * that is being collected, or the previous completed set.
1519 * <p>
1520 * The current set is the most timely, but may not contain enough data
1521 * to be meaningful if the new interval has just started.
1522 * <p>
1523 * The previous set is complete and thus most likely to have enough samples
1524 * to be useful, but is not completely current.
1525 * <p>
1526 * If a "previous" value returned from our local store is not marked
1527 * as authoritative, then we convert it to an authoritative value,
1528 * write that back to our local store and return it.
1529 * (The "current" value is generally not authoritative.)
1530 *
1531 * @param def event definition (must be for an event); never null
1532 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
1533 * @param current if true the current event set is returned,
1534 * else the previous complete set is returned
1535 *
1536 * @return requested event set; may be empty but never null if requested set not available
1537 */
1538 public EventVariableValue getEventValue(final SimpleVariableDefinition def,
1539 final EventPeriod intervalSelector,
1540 final boolean current)
1541 {
1542 final EventVariableValue eventValue = varMgr.getEventValue(def, intervalSelector, current);
1543 if(!current && !eventValue.isAuthoritative())
1544 {
1545 final EventVariableValue authValue = eventValue.makeAuthoritative();
1546 varMgr.setEventValue(authValue);
1547 return(authValue);
1548 }
1549 return(eventValue);
1550 }
1551
1552 /**Get the specified event sets for the specified intervals; never null.
1553 * This allows retrieval of zero or more event sets for the specified
1554 * interval size.
1555 * <p>
1556 * Requests for more than SystemVariables.EVENT_SAMPLES_RETAINED in the
1557 * past (or for the future!) cannot be satisfied and data will not be
1558 * returned for them.
1559 * <p>
1560 * Usually not more than SystemVariables.EVENT_SAMPLES_RETAINED samples
1561 * will be returned in response to any one request as a safety measure.
1562 * <p>
1563 * (An implementation that is not an end-point may go upstream to fetch
1564 * missing values and cache them to satisfy future requests.)
1565 * <p>
1566 * If a "previous" value (older than the current interval)
1567 * returned from our local store is null or not marked
1568 * as authoritative, then we convert it to an authoritative value,
1569 * write that back to our local store and return it.
1570 * (The "current" value is generally not authoritative,
1571 * and entries in the future cannot be so.)
1572 *
1573 * @param def event definition (must be for an event); never null
1574 * @param intervalSelector one of EVENT_INTERVAL_SELECTOR_xxx values
1575 * @param intervalNumber a time (as from System.currentTimeMillis())
1576 * which identifies the first interval for which data is potentially
1577 * required; if too far in the past or future then possibly no data
1578 * will be available,
1579 * zero is used to access the "all" bucket
1580 * @param whichValues each true bit represents a slot for which data is
1581 * required, bit 0 indicating data from the slot within which
1582 * firstIntervalTime is located, bit 1 the previous slot, etc
1583 *
1584 * @return as many of the requested values as available,
1585 * at least long enough to return all the available values,
1586 * with [0] corresponding to bit 0 in the BitSet;
1587 * may contain nulls or be zero-length but is never null
1588 */
1589 public EventVariableValue[] getEventValues(final SimpleVariableDefinition def,
1590 final EventPeriod intervalSelector,
1591 final long intervalNumber,
1592 final BitSet whichValues)
1593 {
1594 // Compute currentInterval before getting values
1595 // to avoid a race which might make wrongly capture
1596 // an incomplete "current" value as authoritative and complete.
1597 final long currentInterval = intervalSelector.getIntervalNumber(System.currentTimeMillis());
1598
1599 synchronized(varMgr) // Avoid races with new events coming in...
1600 {
1601 EventVariableValue[] result = varMgr.getEventValues(def, intervalSelector, intervalNumber, whichValues);
1602
1603 // Ensure result array is large enough,
1604 // else copy into large enough one.
1605 final int wvl = (whichValues == null) ? 1 : whichValues.length();
1606 if(result.length < wvl)
1607 {
1608 final EventVariableValue newResult[] = new EventVariableValue[wvl];
1609 System.arraycopy(result, 0, newResult, 0, result.length);
1610 result = newResult;
1611 }
1612
1613 // For any values newer than the current interval,
1614 // and for which our local store had missing or non-authoritative values,
1615 // convert them to authoritative values
1616 // so that upstream users can cache them permanently.
1617 BitSet wv = whichValues;
1618 if(wv == null)
1619 {
1620 // Create synthetic BitSet for simplicity below.
1621 wv = new BitSet(1);
1622 wv.set(0);
1623 }
1624 for(int i = wv.nextSetBit(0); i >= 0; i = wv.nextSetBit(i+1))
1625 {
1626 final long slotIntervalNumber = intervalNumber - i;
1627 // We cannot make current/future events authoritative.
1628 if(slotIntervalNumber >= currentInterval)
1629 { continue; }
1630
1631 final EventVariableValue evv = result[i];
1632 if((evv == null) || !evv.isAuthoritative())
1633 {
1634 // Make value authoritative,
1635 // creating a new empty one if need be.
1636 final EventVariableValue authValue = (evv != null) ? evv.makeAuthoritative() :
1637 new EventVariableValue(true,
1638 def,
1639 intervalSelector,
1640 slotIntervalNumber,
1641 0, null, null);
1642
1643 assert(authValue.isAuthoritative());
1644 assert((evv == null) || (authValue.getIntervalNumber() == evv.getIntervalNumber()));
1645 assert((evv == null) || (authValue.getDef().equals(evv.getDef())));
1646
1647 // Return authoritative value to caller.
1648 result[i] = authValue;
1649
1650 // Save in our local store.
1651 varMgr.setEventValue(authValue);
1652
1653 //if(IsDebug.isDebug) { System.out.println("Storing authoritative " + authValue); }
1654 }
1655 }
1656
1657 return(result);
1658 }
1659 }
1660
1661
1662 /**Creates the name of a thumbnail starting at the thumbnail base directory from a full exhibit name.
1663 */
1664 private static String thumbnailRelPath(final Name.ExhibitFull exhibitName)
1665 {
1666 // Construct the name of the thumbnails file if it exists.
1667 return(ExhibitName.getDirComponent(exhibitName).toString() + File.separatorChar +
1668 ExhibitName.getFileComponent(exhibitName) + THUMBNAIL_SUFFIX);
1669 }
1670
1671 // /**Creates thumbnails from all exhibits possible.
1672 // * This is used to create exhibit thumbnails off-line,
1673 // * taking as long and as much memory as required
1674 // * and processing the resource-intensive thumbnail building sequentially.
1675 // * <p>
1676 // * These luxuries of non-real-time and lots of memory and CPU are denied
1677 // * to the WAR/EAR servers when they try to compute thumbnails on the fly,
1678 // * since there may be many simultaneous users and even attempted
1679 // * Denial-of-Service attacks.
1680 // * <p>
1681 // * The thumbnails created here can be loaded by master servers,
1682 // * and through it any slaves, thus making a much wider range of thumbnails
1683 // * available than could be computed on the fly.
1684 // * <p>
1685 // * We will protest but terminate without an exception if the
1686 // * thumbnail directory does not exist; we will assume that precomputed
1687 // * thumbnails are not required.
1688 // * <p>
1689 // * We will only attempt to (*re) compute a thumbnail if:
1690 // * <ul>
1691 // * <li>The exhibit type is one that we can compute thumbnails for.
1692 // * <li>The thumbnail file does not exist or is older than the exhibit file.
1693 // * </ul>
1694 // * <p>
1695 // * This will attempt to put thumbnail files where the getThumbnails()
1696 // * routine will expect to find them.
1697 // */
1698 // private static void createThumbnails(final AllExhibitProperties aep,
1699 // final long stopBy)
1700 // throws IOException
1701 // {
1702 // // Compute full path from exhibit directory.
1703 // final File topDir = (new File(LocalProps.getDataDir(), LocalProps.getThumbnailRelDir())).getCanonicalFile();
1704 //
1705 // // If the top directory for the thumbnails does not exist,
1706 // // then we definitely will not be returning thumbnails.
1707 // // We also warn when this directory does not exist.
1708 // if(!topDir.isDirectory() || !topDir.canWrite())
1709 // {
1710 // // We might wish to limit frequency with which we report missing directory!
1711 // System.err.println("Warning: top-level thumbnail directory absent: " + topDir);
1712 // System.err.println("Aborting creation of thumbnails.");
1713 // return;
1714 // }
1715 //
1716 // // Make an instance of the data retriever for use by the thumbnail generator.
1717 // final ExhibitDataFileSource edfs = new ExhibitDataFileSource(aep);
1718 // final AllExhibitProperties.ExhibitDataSource eds = edfs.makeExhibitDataSource();
1719 //
1720 // // Now examine each exhibit in turn
1721 // // (in sorted order to try to approximate a disc-friendly breadth-first search)
1722 // // and if it is possible to build thumbnails
1723 // // and they don't exist or are older than the exhibit,
1724 // // try to build the thumbnails and save them.
1725 // //
1726 // // This tries to be robust and avoid blowing up if a single thumbnail build has problems.
1727 // final List<Name.ExhibitFull> names = aep.aeid.getAllExhibitNamesSorted();
1728 //
1729 // // Queue of tasks in progress.
1730 // final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1731 //
1732 // for(int i = names.size(); (--i >= 0) && (System.currentTimeMillis() <= stopBy); )
1733 // {
1734 // final Name.ExhibitFull name = names.get(i);
1735 //
1736 // // In the simple case where we know that thumbnails
1737 // // definitely can't be created
1738 // // (from the type, or because there is no exhibit)
1739 // // return the NO_THUMBNAILS value immediately,
1740 // // ignoring the create parameter.
1741 //
1742 // final ExhibitStaticAttr esa = aep.aeid.getStaticAttr(name);
1743 // if(esa == null)
1744 // { continue; }
1745 //
1746 // final ExhibitMIME.ExhibitTypeParameters type = (ExhibitMIME.getInputFileType(esa.getCharSequence()));
1747 // if((type == null) || (type.handler == null) ||
1748 // !type.canPossiblyCreateThumbnailOfSameMIMEType())
1749 // { continue; }
1750 //
1751 // // These tasks are assumed to be heavy on CPU, and not mainly I/O-bound.
1752 // tasks.add(ThreadUtils.computeIntensiveThreadPool.submit(new Runnable(){
1753 // public final void run()
1754 // {
1755 // // Construct the name of the thumbnails file if it exists.
1756 // final String relPath = thumbnailRelPath(name);
1757 //
1758 // // If the thumbnail file seems to exist and is readable,
1759 // // and is newer than the exhibit,
1760 // // then assume that it is up-to-date.
1761 // final File fullPath = new File(topDir, relPath);
1762 // if(fullPath.exists() && fullPath.canRead() && fullPath.isFile() &&
1763 // (fullPath.lastModified() > esa.timestamp))
1764 // { return; }
1765 //
1766 // // OK, looks like we need to (re)build the thumbnails...
1767 // try {
1768 // // Ensure that the destination directory exists.
1769 // final File targetDir = fullPath.getParentFile();
1770 // if(!targetDir.exists())
1771 // { targetDir.mkdirs(); }
1772 //
1773 // // OK, attempt to generate thumbnails here, not under any lock.
1774 // final ExhibitThumbnails tns = type.handler.makeThumbnails(
1775 // esa, eds, aep,
1776 // true); // Ignore resource limitations; this our best chance!
1777 //
1778 // // Attempt to save thumbnails (atomically), without additional compression.
1779 // FileTools.serialiseToFile(tns, fullPath, THUMBNAILS_ARE_GZIPPED, false);
1780 // }
1781 // catch(final ThreadDeath e)
1782 // { throw e; } // Don't intercept this one...
1783 // catch(final Throwable e)
1784 // {
1785 // System.err.println("ERROR: unable to build thumbnails for: " + name);
1786 // e.printStackTrace();
1787 // return; // Attempt to continue.
1788 // }
1789 // }
1790 // }));
1791 //
1792 // // Optimisation: release resources ASAP from any early completed tasks.
1793 // Future<?> headTask;
1794 // while(null != (headTask = tasks.peek()))
1795 // { if(headTask.isDone()) { tasks.remove(); } }
1796 // }
1797 //
1798 // // Wait for all remaining tasks.
1799 // while(!tasks.isEmpty())
1800 // {
1801 // try { tasks.remove().get(); }
1802 // catch(final Exception e)
1803 // {
1804 // final IOException err = new IOException("did not complete");
1805 // err.initCause(e);
1806 // throw err;
1807 // }
1808 // }
1809 // }
1810
1811 /**Computes an AllExhibitProperties object and saves it to outputFile.
1812 * The object is saved serialised and GZIPed.
1813 * <p>
1814 * The save is avoided if we are not forcing a complete recompute
1815 * and the exhibit hash has not changed,
1816 * ie we try to avoid unnecessarily churning the filesystem
1817 * and/or the master server.
1818 * <p>
1819 * It almost certainly does not make sense for the
1820 * WAR_SYSPROPNAME_WARONLY_STATICCACHEFILE property to be set;
1821 * you could get a similar result by copying the file.
1822 * <p>
1823 * In passing, this may update ancillary state such as computable properties
1824 * and thumbnails.
1825 * @param logger TODO
1826 * @param outputFile desired location of cache file; not null
1827 * @param quick if true, attempt to be as quick as possible,
1828 * else if false, magic numbers of exhibits are checked and
1829 * other extra-careful checking is done;
1830 * this should be the default usage
1831 * @param recompute if true, recompute all derived data from scratch,
1832 * eg do not reload any extant cache file
1833 *
1834 * @return AllExhibitProperties (non-null)
1835 */
1836 private static AllExhibitProperties createStaticCacheFile(final SimpleLoggerIF logger,
1837 final File outputFile,
1838 final boolean quick,
1839 final boolean recompute,
1840 final boolean fixaccessions)
1841 throws IOException
1842 {
1843 // final long startTime = System.currentTimeMillis();
1844
1845 // Submit task to update accessions concurrently.
1846 final Future<?> accessionsUpdate = ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1847 public final void run()
1848 {
1849 try
1850 {
1851 // Get a simple listing of exhibits on disc.
1852 final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
1853 LocalProps.getDataDir(),
1854 -1,
1855 !quick);
1856 assert(aeid != null);
1857
1858 // Create/fix accessions files first.
1859 createAccessionFiles(aeid);
1860 }
1861 catch(final IOException e)
1862 {
1863 e.printStackTrace();
1864 throw new Error("failed to create accessions", e);
1865 }
1866 }
1867 });
1868
1869 // If we're being quick then we can't be too careful...
1870 final boolean careful = !quick;
1871
1872 // Wait for accessions update to complete
1873 // so that we can incorporate any new files in the AEP.
1874 try { accessionsUpdate.get(); }
1875 catch(final Exception e) { throw new Error(e); }
1876
1877 // Create an instance.
1878 final ExhibitDataFileSource edfs = new ExhibitDataFileSource(logger);
1879
1880 // Now create and save the cache (if changed)...
1881 System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): started: " +(new Date())+ ".]");
1882 // Get the properties...
1883 // Force reconstruction of AEP from filesystem.
1884 final AllExhibitProperties aep = edfs._getAllExhibitProperties(-1L, careful, true);
1885 System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): *** exhibit count: " + aep.aeid.size() + ".]");
1886
1887 return(aep);
1888 }
1889
1890 /**Create accessions files.
1891 * @return true if any accessions files were created or fixed
1892 */
1893 private static boolean createAccessionFiles(final AllExhibitImmutableData aeid)
1894 throws IOException
1895 {
1896 final AtomicBoolean result = new AtomicBoolean(); // No updates/fixes/creations yet...
1897 final AtomicReference<String> couldNotMakeAccession = new AtomicReference<String>();
1898
1899 final String dataDir = LocalProps.getDataDir();
1900
1901 // Check accession files exist for all exhibits.
1902 logFSAccess("Adding accession files as necessary...", false);
1903
1904 // Queue of tasks yet to complete...
1905 final Queue<Future<?>> tasks = new LinkedBlockingQueue<Future<?>>();
1906
1907 // Do this in approximately sorted order so as to improve disc throughput...
1908 for(final Name.ExhibitFull exhibitName : aeid.getAllExhibitNamesSorted())
1909 {
1910 // These tasks are assumed to be light on CPU, and I/O-bound.
1911 tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
1912 public final void run()
1913 {
1914 try
1915 {
1916 final String newAccessionFilename = ExhibitPropsLoadable.relPathToNewAccession(exhibitName);
1917 final File newAccessionFile = new File(dataDir, newAccessionFilename);
1918 if(!newAccessionFile.exists())
1919 {
1920 // Compute the accession data...
1921 final AccessionData ad = AccessionData.fromExhibitFile(new File(dataDir, exhibitName.toString()));
1922 // Convert to (terse) XML.
1923 final String xml = TextUtils.toXML(ad.getAsDOM(), false, true);
1924 // Convert to UTF-8 text bytes ready to save.
1925 final byte utf8[] = xml.getBytes(CoreConsts.FILE_ENCODING_UTF_8);
1926 // Save new accessions file as near atomically as possible.
1927 final String nAFS = newAccessionFile.toString();
1928 logFSAccess("Creating accession file: "+nAFS, false);
1929 if(FileTools.replacePublishedFile(nAFS, utf8, false))
1930 { result.set(true); }
1931 }
1932 }
1933 catch(final IOException e)
1934 {
1935 // Whinge but continue if we encounter an error.
1936 e.printStackTrace();
1937 couldNotMakeAccession.set("IOException inspecting/making accession file for: " + exhibitName + ": " + e.getMessage());
1938 }
1939 }
1940 }));
1941
1942 // Optimisation: release resources ASAP from any early completed tasks.
1943 // NOTE: handles exceptions differently than final mop-up which may be undesirable.
1944 Future<?> headTask;
1945 while(null != (headTask = tasks.peek()))
1946 { if(headTask.isDone()) { tasks.remove(); } }
1947 }
1948
1949 // Wait for all remaining tasks.
1950 while(!tasks.isEmpty())
1951 {
1952 try { tasks.remove().get(); }
1953 catch(final Exception e)
1954 {
1955 final IOException err = new IOException("did not complete");
1956 err.initCause(e);
1957 throw err;
1958 }
1959 }
1960
1961 if(couldNotMakeAccession.get() != null)
1962 {
1963 System.err.println("ERROR: could not make at least one missing (new) accession file: ");
1964 System.err.println(" REASON: " + couldNotMakeAccession);
1965 }
1966
1967 return(result.get());
1968 }
1969
1970 /**Get requested Properties selected by key and versionID.
1971 * Fetches a Properties set unconditionally (versionID == -1)
1972 * else if the versionID presented is not current.
1973 *
1974 * @param key selector (with possible embedded sub-key)
1975 * for desired properties set; never null
1976 * @param versionID if -1 then map is always returned if available,
1977 * else must be non-negative and null is returned if the versionID
1978 * presented matches that of the current version
1979 * (ie if the caller has presumably got the up-to-date version);
1980 * may be a timestamp or a hash or other value,
1981 * and by convention is zero only for an empty properties set
1982 *
1983 * @return null, or Properties map guaranteed to contain only
1984 * String keys and values
1985 */
1986 public java.util.Properties getProperties(final PropsKey key,
1987 final long versionID)
1988 throws IOException
1989 {
1990 throw new IOException("NOT IMPLEMENTED");
1991 }
1992
1993
1994 /**Run from the command-line with a single argument (the output file name).
1995 * There is an optional <code>-quick</code> first argument,
1996 * which if present which attempts to cap the time spent in one run
1997 * (typically to no more than a few minutes)
1998 * so that any work to be done can be done incrementally
1999 * in several passes if need be.
2000 * <p>
2001 * When making a new static cache file we always try to reload the old one
2002 * (unless the <code>-recompute</code> flag is set, in which case we will not load it)
2003 * to save lots of time (especially for the computed properties)
2004 * and possibly preserve the hashNotChangedSince value.
2005 * <p>
2006 * If the <code>-fixaccessions</code> flag is passed then we try to fix existing
2007 * accessions files that are missing information, eg new checksum types,
2008 * though we will never try to "fix" an incorrect checksum for example
2009 * (that will always result in a warning on the console.)
2010 * <p>
2011 * If the <code>-checkexhibits</code> flag is true
2012 * then we try to verify that each exhibit's exhibit length, timestamp and hash
2013 * matches the values captured in the associated accession file.
2014 * We may check additional data (eg FEC data) in future.
2015 * Note that this option overrides any other options,
2016 * and avoids writing anything to disc.
2017 */
2018 public static void main(final String args[])
2019 {
2020 final long startTime = System.currentTimeMillis();
2021
2022 if(args.length < 1)
2023 {
2024 System.err.println("Expects: " +
2025 "[-quick | -recompute | -fixaccessions | -checkexhibits ] " +
2026 "filename");
2027 System.exit(1);
2028 }
2029
2030 boolean quick = false; // If true, use quicker exhibit-collection method.
2031 boolean recompute = false; // If true, force recompute from scratch; do not reload old cache file.
2032 boolean fixaccessions = false; // If true, fix existing accessions files.
2033 boolean checkexhibits = false; // If true, verify exhibit files are not corrupt.
2034 for(int i = args.length - 1; --i >= 0; )
2035 {
2036 if("-quick".equals(args[i])) { quick = true; }
2037 else if("-recompute".equals(args[i])) { recompute = true; }
2038 else if("-fixaccessions".equals(args[i])) { fixaccessions = true; }
2039 else if("-checkexhibits".equals(args[i])) { checkexhibits = true; }
2040 else
2041 {
2042 System.err.println("Unknown flag '" + args[i] + "'.");
2043 System.exit(1);
2044 return;
2045 }
2046 }
2047
2048 System.out.println("[ExhibitDataFileSource: START: " +(new Date(startTime))+ ".]");
2049
2050 // Get the filename to load/save from/to; always last.
2051 final File staticCacheFilename = new File(args[args.length - 1]);
2052
2053 try {
2054 if(checkexhibits)
2055 {
2056 checkExhibitData(quick, staticCacheFilename.getParentFile());
2057 }
2058 else
2059 {
2060 // Create static cache file and related state.
2061 final AllExhibitProperties aep = createStaticCacheFile(GenUtils.systemOutLogger, staticCacheFilename, quick, recompute, fixaccessions);
2062 assert(aep != null);
2063
2064 // // Create any thumbnails that are missing.
2065 // System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building starting: " +(new Date())+ ".]");
2066 // final long timeSoFar = Math.max(30000, 7 * (System.currentTimeMillis() - startTime));
2067 // createThumbnails(aep, (!quick) ? Long.MAX_VALUE : (System.currentTimeMillis() + timeSoFar));
2068 // System.out.println("[ExhibitDataFileSource.createStaticCacheFile(): thumbnail building finished: " +(new Date())+ ".]");
2069 }
2070
2071 final long endTime = System.currentTimeMillis();
2072 final long runTime = endTime - startTime;
2073 System.out.println("[ExhibitDataFileSource: END: " +(new Date(endTime))+ ": "+runTime+"ms.]");
2074 }
2075 catch(final Exception e)
2076 {
2077 e.printStackTrace();
2078 System.exit(1); // Exit with error code.
2079 }
2080 }
2081
2082 /**Check exhibit data for corruption.
2083 * @param quick take some optional speed-ups
2084 * @throws IOException in case of difficulty
2085 */
2086 private static void checkExhibitData(final boolean quick,
2087 final File dataDir)
2088 throws IOException
2089 {
2090 System.out.println("Checking exhibit data... (cache file ignored)");
2091 final File baseDir = (dataDir != null) ? dataDir : new File(LocalProps.getDataDir());
2092 // Get a simple listing of exhibits on disc.
2093 final AllExhibitImmutableData aeid = _getAllExhibitImmutableData(
2094 baseDir.getPath(),
2095 -1,
2096 !quick);
2097 assert(aeid != null);
2098 System.out.println("Exhibits found: " + aeid.length);
2099
2100 // Collect existing accession data (as part of the loaded properties),
2101 // and recompute the hashes from scratch to compare,
2102 // and make sure that nothing has changed.
2103 // We do this in parallel to maximise I/O throughput,
2104 // and use a ConcurrentHashMap as our working store as it is thread-safe.
2105 // We get these using our non-CPU-bound thread pool
2106 // and wait for them all to complete.
2107 final List<Name.ExhibitFull> sortedNames = aeid.getAllExhibitNamesSorted();
2108 final Map<Name.ExhibitFull,AccessionData> accessionMap = new ConcurrentHashMap<Name.ExhibitFull, AccessionData>(aeid.size() * 2);
2109 final List<Future<?>> tasks = new ArrayList<Future<?>>(sortedNames.size());
2110 for(final Name.ExhibitFull name : sortedNames)
2111 {
2112 tasks.add(ThreadUtils.nonCPUThreadPool.submit(new Runnable(){
2113 public final void run()
2114 {
2115 // We don't store null/EMPTY values.
2116 try
2117 {
2118 final ExhibitPropsLoadable epl = ExhibitPropsLoadable.getLoadableProperties(name, baseDir);
2119 if((epl != null) && !epl.equals(ExhibitPropsLoadable.EMPTY) && (epl.getAccessionMetadata() != null))
2120 {
2121 final AccessionData accessionMetadata = epl.getAccessionMetadata();
2122 accessionMap.put(name, accessionMetadata);
2123
2124 // Compare the accession timestamp and length with the current file.
2125 final ExhibitStaticAttr esa = aeid.getStaticAttr(name);
2126 if(esa == null)
2127 { throw new Error("Missing exhibit: "+name); }
2128 if(accessionMetadata.date == null)
2129 { System.err.println("Accession timestamp missing for: " + name); }
2130 else if(accessionMetadata.date.longValue() != esa.timestamp)
2131 { System.err.println("Accession timestamp does not match current file for: " + name + ": was/now " + new Date(accessionMetadata.date.longValue()) + '/' + new Date(esa.timestamp)); }
2132 if(accessionMetadata.size == null)
2133 { System.err.println("Accession length missing for: " + name); }
2134 else if(accessionMetadata.size.longValue() != esa.length)
2135 { System.err.println("Accession length does not match current file for: " + name + ": was/now " + accessionMetadata.size + '/' + esa.length); }
2136
2137 // Now recompute and compare the accession data hashes.
2138 final InputStream is = new BufferedInputStream(new FileInputStream(new File(baseDir, name.toString())));
2139 try
2140 {
2141 final Tuple.Pair<Integer,ROByteArray> hashes = AccessionData.computeFullFileHashes(is);
2142 if(!hashes.first.equals(accessionMetadata.hashCRC32))
2143 { System.err.println("Accession hash CRC32 changed for: " + name); }
2144 if(!hashes.second.equals(accessionMetadata.hashMD5))
2145 { System.err.println("Accession hash MD5 changed for: " + name); }
2146 }
2147 finally { is.close(); }
2148 }
2149 else
2150 { System.err.println("No accession data available for: " + name); }
2151 }
2152 catch(final Exception e)
2153 { throw new RuntimeException("Error loading accession data for: " + name, e); }
2154 }
2155 }));
2156 }
2157 // Wait for all tasks to finish...
2158 for(final Future<?> task : tasks)
2159 {
2160 try { task.get(); }
2161 catch(final InterruptedException e)
2162 {
2163 final InterruptedIOException err = new InterruptedIOException(e.getMessage());
2164 err.initCause(e);
2165 throw err;
2166 }
2167 catch(final ExecutionException e)
2168 {
2169 final IOException err = new IOException(e.getMessage());
2170 err.initCause(e);
2171 throw err;
2172 }
2173 }
2174 System.out.println("Accession data loaded for exhibits: " + accessionMap.size());
2175 if(accessionMap.size() != aeid.length)
2176 { throw new IOException("Missing (or changed) accession data."); }
2177 }
2178
2179 /**Assumes that this instance is the root/master and so returns Stratum.ROOT; non-null. */
2180 public Stratum getStratum() { return(Stratum.ROOT); }
2181
2182 /**Shut down the data pipeline.
2183 * Saves any pending state (eg log entries and variables)
2184 * to disc if possible.
2185 * <p>
2186 * Has no upstream components to shut down
2187 * nor significant resources (memory) to release.
2188 */
2189 /* @Override */ public void destroy()
2190 {
2191 _saveSystemVariables(varMgr, true); // Force immediate save.
2192 }
2193 }