001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.List; 023import java.util.Map; 024 025import org.eclipse.january.DatasetException; 026import org.eclipse.january.IMonitor; 027import org.eclipse.january.io.ILazyLoader; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.OriginMetadata; 031import org.eclipse.january.metadata.Reshapeable; 032import org.eclipse.january.metadata.Sliceable; 033import org.eclipse.january.metadata.Transposable; 034 035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 036 private static final long serialVersionUID = 2467865859867440242L; 037 038 protected int[] oShape; // original shape 039 protected long size; // number of items 040 protected int dtype; // dataset type 041 protected int isize; // number of elements per item 042 043 protected ILazyLoader loader; 044 protected LazyDataset base = null; // used for transpose 045 046 // relative to loader or base 047 protected int prepShape = 0; // prepending and post-pending 048 protected int postShape = 0; // changes to shape 049 protected int[] begSlice = null; // slice begin 050 protected int[] delSlice = null; // slice delta 051 protected int[] map; // transposition map (same length as current shape) 052 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 053 054 /** 055 * Create a lazy dataset 056 * @param name 057 * @param dtype dataset type 058 * @param elements 059 * @param shape 060 * @param loader 061 */ 062 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 063 this.name = name; 064 this.shape = shape.clone(); 065 this.oShape = this.shape; 066 this.loader = loader; 067 this.dtype = dtype; 068 this.isize = elements; 069 try { 070 size = ShapeUtils.calcLongSize(shape); 071 } catch (IllegalArgumentException e) { 072 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 073 } 074 } 075 076 /** 077 * Create a lazy dataset 078 * @param name 079 * @param dtype dataset type 080 * @param shape 081 * @param loader 082 */ 083 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 084 this(name, dtype, 1, shape, loader); 085 } 086 087 /** 088 * Create a lazy dataset based on in-memory data (handy for testing) 089 * @param dataset 090 */ 091 public static LazyDataset createLazyDataset(final Dataset dataset) { 092 return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(), 093 new ILazyLoader() { 094 private static final long serialVersionUID = -6725268922780517523L; 095 096 final Dataset d = dataset; 097 @Override 098 public boolean isFileReadable() { 099 return true; 100 } 101 102 @Override 103 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 104 return d.getSlice(mon, slice); 105 } 106 }); 107 } 108 109 /** 110 * Can return -1 for unknown 111 */ 112 @Override 113 public int getDType() { 114 return dtype; 115 } 116 117 /** 118 * Can return -1 for unknown 119 */ 120 @Override 121 public int getElementsPerItem() { 122 return isize; 123 } 124 125 @Override 126 public int getSize() { 127 return (int) size; 128 } 129 130 @Override 131 public String toString() { 132 StringBuilder out = new StringBuilder(); 133 134 if (name != null && name.length() > 0) { 135 out.append("Lazy dataset '"); 136 out.append(name); 137 out.append("' has shape ["); 138 } else { 139 out.append("Lazy dataset shape is ["); 140 } 141 int rank = shape == null ? 0 : shape.length; 142 143 if (rank > 0 && shape[0] >= 0) { 144 out.append(shape[0]); 145 } 146 for (int i = 1; i < rank; i++) { 147 out.append(", " + shape[i]); 148 } 149 out.append(']'); 150 151 return out.toString(); 152 } 153 154 @Override 155 public boolean equals(Object obj) { 156 if (!super.equals(obj)) 157 return false; 158 159 LazyDataset other = (LazyDataset) obj; 160 if (dtype != other.dtype) { 161 return false; 162 } 163 if (isize != other.isize) { 164 return false; 165 } 166 167 if (!Arrays.equals(shape, other.shape)) { 168 return false; 169 } 170 171 if (loader != other.loader) { 172 return false; 173 } 174 175 if (prepShape != other.prepShape) { 176 return false; 177 } 178 179 if (postShape != other.postShape) { 180 return false; 181 } 182 183 if (!Arrays.equals(begSlice, other.begSlice)) { 184 return false; 185 } 186 if (!Arrays.equals(delSlice, other.delSlice)) { 187 return false; 188 } 189 if (!Arrays.equals(map, other.map)) { 190 return false; 191 } 192 return true; 193 } 194 195 @Override 196 public LazyDataset clone() { 197 LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader); 198 ret.shape = shape; 199 ret.size = size; 200 ret.prepShape = prepShape; 201 ret.postShape = postShape; 202 ret.begSlice = begSlice; 203 ret.delSlice = delSlice; 204 ret.map = map; 205 ret.base = base; 206 ret.metadata = copyMetadata(); 207 ret.oMetadata = oMetadata; 208 return ret; 209 } 210 211 @Override 212 public void setShape(int... shape) { 213 setShapeInternal(shape); 214 } 215 216 @Override 217 public LazyDataset squeezeEnds() { 218 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 219 return this; 220 } 221 222 @Override 223 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 224 return getSlice(null, start, stop, step); 225 } 226 227 @Override 228 public Dataset getSlice(Slice... slice) throws DatasetException { 229 if (slice == null || slice.length == 0) { 230 return getSlice(null, new SliceND(shape)); 231 } 232 return getSlice(null, new SliceND(shape, slice)); 233 } 234 235 @Override 236 public Dataset getSlice(SliceND slice) throws DatasetException { 237 return getSlice(null, slice); 238 } 239 240 @Override 241 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 242 if (slice == null || slice.length == 0) { 243 return getSlice(monitor, new SliceND(shape)); 244 } 245 return getSlice(monitor, new SliceND(shape, slice)); 246 } 247 248 @Override 249 public LazyDataset getSliceView(Slice... slice) { 250 if (slice == null || slice.length == 0) { 251 return getSliceView(new SliceND(shape)); 252 } 253 return getSliceView(new SliceND(shape, slice)); 254 } 255 256 /** 257 * @param nShape 258 */ 259 private void setShapeInternal(int... nShape) { 260 261 long nsize = ShapeUtils.calcLongSize(nShape); 262 if (nsize != size) { 263 throw new IllegalArgumentException("Size of new shape is not equal to current size"); 264 } 265 266 if (nsize == 1) { 267 shape = nShape.clone(); 268 return; 269 } 270 271 int ob = -1; // first non-unit dimension 272 int or = shape.length; 273 for (int i = 0; i < or; i++) { 274 if (shape[i] != 1) { 275 ob = i; 276 break; 277 } 278 } 279 assert ob >= 0; 280 int oe = -1; // last non-unit dimension 281 for (int i = or - 1; i >= ob; i--) { 282 if (shape[i] != 1) { 283 oe = i; 284 break; 285 } 286 } 287 assert oe >= 0; 288 oe++; 289 290 int nb = -1; // first non-unit dimension 291 int nr = nShape.length; 292 for (int i = 0; i < nr; i++) { 293 if (nShape[i] != 1) { 294 nb = i; 295 break; 296 } 297 } 298 299 int i = ob; 300 int j = nb; 301 if (begSlice == null) { 302 for (; i < oe && j < nr; i++, j++) { 303 if (shape[i] != nShape[j]) { 304 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 305 } 306 } 307 } else { 308 int[] nBegSlice = new int[nr]; 309 int[] nDelSlice = new int[nr]; 310 Arrays.fill(nDelSlice, 1); 311 for (; i < oe && j < nr; i++, j++) { 312 if (shape[i] != nShape[j]) { 313 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 314 } 315 nBegSlice[j] = begSlice[i]; 316 nDelSlice[j] = delSlice[i]; 317 } 318 319 begSlice = nBegSlice; 320 delSlice = nDelSlice; 321 } 322 prepShape += nb - ob; 323 postShape += nr - oe; 324 325 storeMetadata(metadata, Reshapeable.class); 326 metadata = copyMetadata(); 327 reshapeMetadata(shape, nShape); 328 shape = nShape; 329 } 330 331 @Override 332 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 333 return getSliceView(new SliceND(shape, start, stop, step)); 334 } 335 336 @Override 337 public LazyDataset getSliceView(SliceND slice) { 338 LazyDataset view = clone(); 339 if (slice.isAll()) { 340 return view; 341 } 342 343 int[] lstart = slice.getStart(); 344 int[] lstep = slice.getStep(); 345 final int rank = shape.length; 346 347 int[] nShape = slice.getShape(); 348 view.shape = nShape; 349 view.size = ShapeUtils.calcLongSize(nShape); 350 if (begSlice == null) { 351 view.begSlice = lstart.clone(); 352 view.delSlice = lstep.clone(); 353 } else { 354 view.begSlice = new int[rank]; 355 view.delSlice = new int[rank]; 356 for (int i = 0; i < rank; i++) { 357 view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i]; 358 view.delSlice[i] = delSlice[i] * lstep[i]; 359 } 360 } 361 view.storeMetadata(metadata, Sliceable.class); 362 363 view.sliceMetadata(true, slice); 364 return view; 365 } 366 367 @Override 368 public LazyDataset getTransposedView(int... axes) { 369 LazyDataset view = clone(); 370 371 // everything now is seen through a map 372 axes = checkPermutatedAxes(shape, axes); 373 if (axes == null) { 374 return view; 375 } 376 377 int r = shape.length; 378 view.shape = new int[r]; 379 for (int i = 0; i < r; i++) { 380 view.shape[i] = shape[axes[i]]; 381 } 382 383 view.prepShape = 0; 384 view.postShape = 0; 385 view.begSlice = null; 386 view.delSlice = null; 387 view.map = axes; 388 view.base = this; 389 view.storeMetadata(metadata, Transposable.class); 390 view.transposeMetadata(axes); 391 return view; 392 } 393 394 @Override 395 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 396 return getSlice(monitor, new SliceND(shape, start, stop, step)); 397 } 398 399 @Override 400 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 401 402 if (loader != null && !loader.isFileReadable()) { 403 return null; // TODO add interaction to use plot (or remote) server to load dataset 404 } 405 406 SliceND nslice = calcTrueSlice(slice); 407 408 Dataset a; 409 if (base != null) { 410 a = base.getSlice(monitor, nslice); 411 } else { 412 try { 413 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 414 } catch (IOException e) { 415 logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 416 Arrays.toString(slice.getStep()), loader), e); 417 throw new DatasetException(e); 418 } 419 a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE); 420 if (metadata != null && a instanceof LazyDatasetBase) { 421 LazyDatasetBase ba = (LazyDatasetBase) a; 422 ba.metadata = copyMetadata(); 423 if (oMetadata != null) { 424 ba.restoreMetadata(oMetadata); 425 } 426 //metadata axis may be larger than data 427 if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) { 428 ba.sliceMetadata(true, nslice); 429 } 430 } 431 } 432 if (map != null) { 433 a = a.getTransposedView(map); 434 } 435 if (slice != null) { 436 a.setShape(slice.getShape()); 437 } 438 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name)); 439 440 return a; 441 } 442 443 // reverse transform 444 private int[] getOriginal(int[] values) { 445 if (values == null) { 446 return null; 447 } 448 int r = values.length; 449 if (map == null || r < 2) { 450 return values; 451 } 452 int[] ovalues = new int[r]; 453 for (int i = 0; i < r; i++) { 454 ovalues[map[i]] = values[i]; 455 } 456 return ovalues; 457 } 458 459 protected final SliceND calcTrueSlice(SliceND slice) { 460 if (slice == null) { 461 slice = new SliceND(shape); 462 } 463 int[] lstart = slice.getStart(); 464 int[] lstop = slice.getStop(); 465 int[] lstep = slice.getStep(); 466 467 int[] nstart; 468 int[] nstop; 469 int[] nstep; 470 471 int r = base == null ? oShape.length : base.shape.length; 472 nstart = new int[r]; 473 nstop = new int[r]; 474 nstep = new int[r]; 475 Arrays.fill(nstop, 1); 476 Arrays.fill(nstep, 1); 477 { 478 int i = 0; 479 int j = 0; 480 if (prepShape < 0) { // ignore entries from new slice 481 i = -prepShape; 482 } else if (prepShape > 0) { 483 j = prepShape; 484 } 485 if (begSlice == null) { 486 for (; i < r && j < shape.length; i++, j++) { 487 nstart[i] = lstart[j]; 488 nstop[i] = lstop[j]; 489 int d = lstep[j]; 490 if (d < 0 && nstop[i] < 0) { // need to wrap around further 491 int l = base == null ? oShape[j]: base.shape[j]; 492 nstop[i] -= l; 493 } 494 nstep[i] = d; 495 } 496 } else { 497 for (; i < r && j < shape.length; i++, j++) { 498 int b = begSlice[j]; 499 int d = delSlice[j]; 500 nstart[i] = b + lstart[j] * d; 501 nstop[i] = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1); 502 if (d < 0 && nstop[i] < 0) { // need to wrap around further 503 int l = base == null ? oShape[j]: base.shape[j]; 504 nstop[i] -= l; 505 } 506 nstep[i] = lstep[j] * d; 507 } 508 } 509 if (map != null) { 510 nstart = getOriginal(nstart); 511 nstop = getOriginal(nstop); 512 nstep = getOriginal(nstep); 513 } 514 } 515 516 return createSlice(nstart, nstop, nstep); 517 } 518 519 protected final IDataset transformInput(IDataset data) { 520 if (map == null) { 521 return data; 522 } 523 return data.getTransposedView(map); 524 } 525 526 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 527 return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep); 528 } 529 530 /** 531 * Store metadata items that has given annotation 532 * @param origMetadata 533 * @param aclazz 534 */ 535 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 536 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 537 if (mclazzes.size() == 0) { 538 return; 539 } 540 541 if (oMetadata == null) { 542 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 543 } 544 for (Class<? extends MetadataType> mc : mclazzes) { 545 if (oMetadata.containsKey(mc)) { 546 continue; // do not overwrite original 547 } 548 549 List<MetadataType> l = origMetadata.get(mc); 550 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 551 for (MetadataType m : l) { 552 nl.add(m.clone()); 553 } 554 oMetadata.put(mc, nl); 555 } 556 } 557 558 @SuppressWarnings("unchecked") 559 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 560 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 561 if (metadata == null) { 562 return mclazzes; 563 } 564 565 for (Class<? extends MetadataType> c : metadata.keySet()) { 566 boolean hasAnn = false; 567 for (MetadataType m : metadata.get(c)) { 568 if (m == null) { 569 continue; 570 } 571 572 Class<? extends MetadataType> mc = m.getClass(); 573 do { // iterate over super-classes 574 for (Field f : mc.getDeclaredFields()) { 575 if (f.isAnnotationPresent(aclazz)) { 576 hasAnn = true; 577 break; 578 } 579 } 580 Class<?> sclazz = mc.getSuperclass(); 581 if (!MetadataType.class.isAssignableFrom(sclazz)) { 582 break; 583 } 584 mc = (Class<? extends MetadataType>) sclazz; 585 } while (!hasAnn); 586 if (hasAnn) { 587 break; 588 } 589 } 590 if (hasAnn) { 591 mclazzes.add(c); 592 } 593 } 594 return mclazzes; 595 } 596 597 /** 598 * Gets the maximum size of a slice of a dataset in a given dimension 599 * which should normally fit in memory. Note that it might be possible 600 * to get more in memory, this is a conservative estimate and seems to 601 * almost always work at the size returned; providing Xmx is less than 602 * the physical memory. 603 * 604 * To get more in memory increase -Xmx setting or use an expression 605 * which calls a rolling function (like rmean) instead of slicing directly 606 * to memory. 607 * 608 * @param lazySet 609 * @param dimension 610 * @return maximum size of dimension that can be sliced. 611 */ 612 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 613 // size in bytes of each item 614 final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem()); 615 616 // Max in bytes takes into account our minimum requirement 617 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 618 619 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 620 // Space specified in bytes per item available 621 final double space = max/lazySet.getSize(); 622 623 // If we have room for this whole dataset, then fine 624 int[] shape = lazySet.getShape(); 625 if (space >= size) { 626 return shape[dimension]; 627 } 628 629 // Otherwise estimate what we can fit in, conservatively. 630 // First get size of one slice, see it that fits, if not, still return 1 631 double sizeOneSlice = size; // in bytes 632 for (int dim = 0; dim < shape.length; dim++) { 633 if (dim == dimension) { 634 continue; 635 } 636 sizeOneSlice *= shape[dim]; 637 } 638 double avail = max / sizeOneSlice; 639 if (avail < 1) { 640 return 1; 641 } 642 643 // We fudge this to leave some room 644 return (int) Math.floor(avail/4d); 645 } 646}