dials.integrate

Introduction

This program is used to integrate the reflections on the diffraction images. It is called with an experiment list outputted from dials.index or dials.refine and a corresponding set of strong spots from which a profile model is calculated. The program will output a set of integrated reflections and an experiment list with additional profile model data. The data can be reintegrated using the same profile model by inputting this integrated.expt file back into dials.integate.

Examples:

dials.integrate models.expt refined.refl

dials.integrate models.expt refined.refl output.reflections=integrated.refl

dials.integrate models.expt refined.refl profile.fitting=False

dials.integrate models.expt refined.refl background.algorithm=glm

Basic parameters

output {
  experiments = 'integrated.expt'
  reflections = 'integrated.refl'
  phil = 'dials.integrate.phil'
  log = 'dials.integrate.log'
  report = None
  include_bad_reference = False
}
scan_range = None
create_profile_model = True
exclude_images = None
integration {
  lookup {
    mask = None
  }
  block {
    size = auto
    units = *degrees radians frames
    threshold = 0.95
    force = False
    max_memory_usage = 0.90
  }
  use_dynamic_mask = True
  debug {
    reference {
      filename = "reference_profiles.refl"
      output = False
    }
    during = modelling *integration
    output = False
    separate_files = True
    delete_shoeboxes = False
    select = None
    split_experiments = True
  }
  profile {
    fitting = True
    validation {
      number_of_partitions = 1
      min_partition_size = 100
    }
  }
  overlaps_filter {
    foreground_foreground {
      enable = False
    }
    foreground_background {
      enable = False
    }
  }
  mp {
    method = *multiprocessing drmaa sge lsf pbs
    njobs = 1
    nproc = 1
    multiprocessing.n_subset_split = None
  }
  summation {
    detector_gain = 1
  }
  background {
    algorithm = *Auto glm gmodel median null simple
    glm {
      robust {
        tuning_constant = 1.345
      }
      model {
        algorithm = constant2d *constant3d loglinear2d loglinear3d
      }
      min_pixels = 10
    }
    gmodel {
      robust {
        algorithm = False
        tuning_constant = 1.345
      }
      min_pixels = 10
      model = None
    }
    median {
    }
    simple {
      outlier {
        algorithm = *null nsigma truncated normal plane tukey
      }
      model {
        algorithm = constant2d *constant3d linear2d linear3d
      }
      min_pixels = 10
    }
  }
  centroid {
    algorithm = *simple
  }
}
profile {
  algorithm = *gaussian_rs
  gaussian_rs {
    scan_varying = False
    min_spots {
      overall = 50
      per_degree = 20
    }
    sigma_m_algorithm = basic *extended
    centroid_definition = com *s1
    parameters {
      n_sigma = 3.0
      sigma_b = None
      sigma_m = None
    }
    filter {
      min_zeta = 0.05
    }
    fitting {
      scan_step = 5
      grid_size = 5
      threshold = 0.02
      grid_method = single *regular_grid circular_grid spherical_grid
      fit_method = *reciprocal_space detector_space
      detector_space {
        deconvolution = False
      }
    }
  }
}
prediction {
  d_min = None
  d_max = None
  margin = 1
  force_static = False
  padding = 1.0
}
absorption_correction {
  apply = False
  algorithm = fuller_kapton kapton_2019 other
  fuller_kapton {
    xtal_height_above_kapton_mm {
      value = 0.02
      sigma = 0.01
    }
    rotation_angle_deg {
      value = 1.15
      sigma = 0.1
    }
    kapton_half_width_mm {
      value = 1.5875
      sigma = 0.5
    }
    kapton_thickness_mm {
      value = 0.05
      sigma = 0.005
    }
    smart_sigmas = False
    within_spot_sigmas = True
  }
}

Full parameter definitions

output {
  experiments = 'integrated.expt'
    .help = "The experiments output filename"
    .type = str
  output_unintegrated_reflections = False
    .help = "Include unintegrated reflections in output file"
    .type = bool
    .expert_level = 2
  reflections = 'integrated.refl'
    .help = "The integrated output filename"
    .type = str
  phil = 'dials.integrate.phil'
    .help = "The output phil file"
    .type = str
  log = 'dials.integrate.log'
    .help = "The log filename"
    .type = str
  report = None
    .help = "The integration report filename (*.xml or *.json)"
    .type = str
  include_bad_reference = False
    .help = "Include bad reference data including unindexed spots, and"
            "reflections whose predictions are messed up in the reflection"
            "table output. Reflections will have the 'bad_reference' flag set."
    .type = bool
}
scan_range = None
  .help = "Explicitly specify the images to be processed. Only applicable when"
          "experiment list contains a single imageset."
  .type = ints(size=2)
  .multiple = True
create_profile_model = True
  .help = "Create the profile model"
  .type = bool
sampling
  .expert_level = 1
{
  reflections_per_degree = 50
    .help = "The number of predicted reflections per degree of the sequence "
            "to integrate."
    .type = float(value_min=0, allow_none=True)
  minimum_sample_size = 1000
    .help = "cutoff that determines whether subsetting of the input "
            "prediction list is done"
    .type = int(allow_none=True)
  maximum_sample_size = None
    .help = "The maximum number of predictions to integrate. Overrides"
            "reflections_per_degree if that produces a larger sample size."
    .type = int(value_min=1, allow_none=True)
  integrate_all_reflections = True
    .help = "Override reflections_per_degree and integrate all predicted"
            "reflections."
    .type = bool
  random_seed = 0
    .help = "Random seed for sampling"
    .type = int(allow_none=True)
}
exclude_images = None
  .help = "Exclude images from integration (e.g. 1,2,3,4,5 etc)"
  .type = ints
integration {
  lookup
    .help = "Parameters specifying lookup file path"
  {
    mask = None
      .help = "The path to the mask file."
      .type = str
  }
  block {
    size = auto
      .help = "The block size in rotation angle (degrees)."
      .type = float(allow_none=True)
    units = *degrees radians frames
      .help = "The units of the block size"
      .type = choice
    threshold = 0.95
      .help = "For block size auto the block size is calculated by sorting"
              "reflections by the number of frames they cover and then"
              "selecting the block size to be 2*nframes[threshold] such that"
              "100*threshold % of reflections are guaranteed to be fully"
              "contained in 1 block"
      .type = float(value_min=0, value_max=1, allow_none=True)
    force = False
      .help = "If the number of processors is 1 and force is False, then the"
              "number of blocks may be set to 1. If force is True then the"
              "block size is always calculated."
      .type = bool
    max_memory_usage = 0.90
      .help = "The maximum percentage of available memory to use for"
              "allocating shoebox arrays."
      .type = float(value_min=0, value_max=1, allow_none=True)
  }
  use_dynamic_mask = True
    .help = "Use dynamic mask if available"
    .type = bool
  debug {
    reference {
      filename = "reference_profiles.refl"
        .help = "The filename for the reference profiles"
        .type = str
      output = False
        .help = "Save the reference profiles"
        .type = bool
    }
    during = modelling *integration
      .help = "Do debugging during modelling or integration"
      .type = choice
    output = False
      .help = "Save shoeboxes after each processing task."
      .type = bool
    separate_files = True
      .help = "If this is true, the shoeboxes are saved in separate files from"
              "the output integrated.refl file. This is necessary in most"
              "cases since the amount of memory used by the shoeboxes is"
              "typically greater than the available system memory. If,"
              "however, you know that memory is not an issue, you can saved"
              "the shoeboxes in the integrated.refl file by setting this"
              "option to False. This only works if the debug output is during"
              "integrated and not modelling."
      .type = bool
    delete_shoeboxes = False
      .help = "Delete shoeboxes immediately before saving files. This option"
              "in combination with debug.output=True enables intermediate"
              "processing steps to make use of shoeboxes."
      .type = bool
    select = None
      .help = "A string specifying the selection. The string should be of the"
              "form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition to"
              "the items in the reflection table, the following implicit"
              "columns are defined if the necessary data is there: "
              "intensity.sum.i_over_sigma  intensity.prf.i_over_sigma"
      .type = reflection_table_selector
    split_experiments = True
      .help = "Split shoeboxes into different files"
      .type = bool
  }
  profile {
    fitting = True
      .help = "Use profile fitting if available"
      .type = bool
    valid_foreground_threshold = 0.75
      .help = "The minimum fraction of foreground pixels that must be valid in"
              "order for a reflection to be integrated by profile fitting."
      .type = float(value_min=0, value_max=1, allow_none=True)
      .expert_level = 2
    validation {
      number_of_partitions = 1
        .help = "The number of subsamples to take from the reference spots. If"
                "the value is 1, then no validation is performed."
        .type = int(value_min=1, allow_none=True)
      min_partition_size = 100
        .help = "The minimum number of spots to use in each subsample."
        .type = int(value_min=1, allow_none=True)
    }
  }
  filter
    .expert_level = 1
  {
    min_zeta = 0.05
      .help = "Filter the reflections by the value of zeta. A value of less"
              "than or equal to zero indicates that this will not be used. A"
              "positive value is used as the minimum permissible value."
      .type = float(value_min=0, value_max=1, allow_none=True)
    ice_rings = False
      .help = "Set the ice ring flags"
      .type = bool
  }
  overlaps_filter {
    foreground_foreground {
      enable = False
        .help = "Remove all spots in which neighbors' foreground impinges on"
                "the spot's foreground"
        .type = bool
    }
    foreground_background {
      enable = False
        .help = "Remove all spots in which neighbors' foreground impinges on"
                "the spot's background"
        .type = bool
    }
  }
  mp {
    method = *multiprocessing drmaa sge lsf pbs
      .help = "The multiprocessing method to use"
      .type = choice
    njobs = 1
      .help = "The number of cluster jobs to use"
      .type = int(value_min=1, allow_none=True)
    nproc = 1
      .help = "The number of processes to use per cluster job"
      .type = int(value_min=1, allow_none=True)
    multiprocessing.n_subset_split = None
      .help = "Number of subsets to split the reflection table for"
              "integration."
      .type = int(value_min=1, allow_none=True)
  }
  summation {
    detector_gain = 1
      .help = "Multiplier for variances after integration of still images. See"
              "Leslie 1999."
      .type = float(allow_none=True)
  }
  background
    .help = "Extensions for background algorithms."
  {
    algorithm = *Auto glm gmodel median null simple
      .help = "The choice of algorithm"
      .type = choice
    glm
      .help = "An extension class implementing a robust GLM background"
              "algorithm."
    {
      robust {
        tuning_constant = 1.345
          .help = "The tuning constant for robust estimation"
          .type = float(allow_none=True)
      }
      model {
        algorithm = constant2d *constant3d loglinear2d loglinear3d
          .help = "The background model to fit"
          .type = choice
      }
      min_pixels = 10
        .help = "The minimum number of pixels required"
        .type = int(value_min=1, allow_none=True)
    }
    gmodel
      .help = "An extension class implementing a robust GLM background"
              "algorithm."
    {
      robust {
        algorithm = False
          .help = "Use the robust algorithm"
          .type = bool
        tuning_constant = 1.345
          .help = "The tuning constant for robust estimation"
          .type = float(allow_none=True)
      }
      min_pixels = 10
        .help = "The minimum number of pixels required"
        .type = int(value_min=1, allow_none=True)
      model = None
        .help = "The model filename"
        .type = str
    }
    median
      .help = "An extension class implementing a median background algorithm."
    {
    }
    simple
      .help = "An extension class implementing simple background subtraction."
    {
      outlier
        .help = "Outlier rejection prior to background fit"
      {
        algorithm = *null nsigma truncated normal plane tukey
          .help = "The outlier rejection algorithm."
          .type = choice
        nsigma
          .help = "Parameters for nsigma outlier rejector"
          .expert_level = 1
        {
          lower = 3
            .help = "Lower n sigma"
            .type = float(allow_none=True)
          upper = 3
            .help = "Upper n sigma"
            .type = float(allow_none=True)
        }
        truncated
          .help = "Parameters for truncated outlier rejector"
          .expert_level = 1
        {
          lower = 0.01
            .help = "Lower bound"
            .type = float(allow_none=True)
          upper = 0.01
            .help = "Upper bound"
            .type = float(allow_none=True)
        }
        normal
          .help = "Parameters for normal outlier rejector"
          .expert_level = 1
        {
          min_pixels = 10
            .help = "The minimum number of pixels to use in calculating the"
                    "background intensity."
            .type = int(allow_none=True)
        }
        plane
          .help = "Parameters for mosflm-like outlier rejector. This algorithm"
                  "is mainly used in conjunction with a linear 2d background."
          .expert_level = 1
        {
          fraction = 1.0
            .help = "The fraction of pixels to use in determining the initial"
                    "plane used for outlier rejection."
            .type = float(allow_none=True)
          n_sigma = 4.0
            .help = "The number of standard deviations above the threshold"
                    "plane to use in rejecting outliers from background"
                    "calculation."
            .type = float(allow_none=True)
        }
        tukey
          .help = "Parameters for tukey outlier rejector"
          .expert_level = 1
        {
          lower = 1.5
            .help = "Lower IQR multiplier"
            .type = float(allow_none=True)
          upper = 1.5
            .help = "Upper IQR multiplier"
            .type = float(allow_none=True)
        }
      }
      model
        .help = "Background model"
      {
        algorithm = constant2d *constant3d linear2d linear3d
          .help = "The choice of background model"
          .type = choice
      }
      min_pixels = 10
        .help = "The minimum number of pixels to compute the background"
        .type = int(value_min=1, allow_none=True)
    }
  }
  centroid
    .help = "Extensions for centroid algorithms."
  {
    algorithm = *simple
      .help = "The choice of algorithm"
      .type = choice
  }
}
profile
  .help = "The interface definition for a profile model."
{
  algorithm = *gaussian_rs
    .help = "The choice of algorithm"
    .type = choice
  gaussian_rs
    .help = "An extension class implementing a reciprocal space gaussian"
            "profile model."
  {
    scan_varying = False
      .help = "Calculate a scan varying model"
      .type = bool
    min_spots
      .help = "if (total_reflections > overall or reflections_per_degree >"
              "per_degree) then do the profile modelling."
    {
      overall = 50
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0, allow_none=True)
      per_degree = 20
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0, allow_none=True)
    }
    sigma_m_algorithm = basic *extended
      .help = "The algorithm to compute mosaicity"
      .type = choice
    centroid_definition = com *s1
      .help = "The centroid to use as beam divergence (centre of mass or s1)"
      .type = choice
    parameters {
      n_sigma = 3.0
        .help = "Sigma multiplier for shoebox"
        .type = float(value_min=0, allow_none=True)
      sigma_b = None
        .help = "Override the sigma_b value (degrees)"
        .type = float(value_min=0, allow_none=True)
      sigma_m = None
        .help = "Override the sigma_m value (degrees)"
        .type = float(value_min=0, allow_none=True)
    }
    filter {
      min_zeta = 0.05
        .help = "Filter reflections by min zeta"
        .type = float(allow_none=True)
    }
    fitting {
      scan_step = 5
        .help = "Space between profiles in degrees"
        .type = float(allow_none=True)
      grid_size = 5
        .help = "The size of the profile grid."
        .type = int(allow_none=True)
      threshold = 0.02
        .help = "The threshold to use in reference profile"
        .type = float(allow_none=True)
      grid_method = single *regular_grid circular_grid spherical_grid
        .help = "Select the profile grid method"
        .type = choice
      fit_method = *reciprocal_space detector_space
        .help = "The fitting method"
        .type = choice
      detector_space {
        deconvolution = False
          .help = "Do deconvolution in detector space"
          .type = bool
      }
    }
  }
}
prediction {
  d_min = None
    .help = "The maximum resolution limit"
    .type = float(allow_none=True)
  d_max = None
    .help = "The minimum resolution limit"
    .type = float(allow_none=True)
  margin = 1
    .help = "The margin to use to scan varying prediction"
    .type = int(allow_none=True)
  force_static = False
    .help = "For scan-varying prediction force scan-static prediction"
    .type = bool
  padding = 1.0
    .help = "The padding in degrees"
    .type = float(value_min=0, allow_none=True)
}
significance_filter
  .expert_level = 1
{
  enable = False
    .help = "If enabled, the significance filter will, for every experiment,"
            "find the highest resolution where the I/sigI remains above a"
            "certain point (controlled by isigi_cutoff)."
    .type = bool
  d_min = None
    .help = "High resolution cutoff for binning. If None, use the highest"
            "resolution reflection as d_min."
    .type = float(allow_none=True)
  n_bins = 20
    .help = "Number of bins to use when examining resolution falloff"
    .type = int(allow_none=True)
  isigi_cutoff = 1.0
    .help = "I/sigI cutoff. Reflections in and past the first bin that falls"
            "below this cutoff will not be retained"
    .type = float(allow_none=True)
}
absorption_correction
  .multiple = True
{
  apply = False
    .help = "must be supplied as a user-defined function with a specific"
            "interface (not documented)"
    .type = bool
  algorithm = fuller_kapton kapton_2019 other
    .help = "a specific absorption correction, or implementation thereof"
            "kapton_2019 is a more general implementation of fuller_kapton for"
            "use on single/multi-panel detectors"
    .type = choice
  fuller_kapton {
    xtal_height_above_kapton_mm {
      value = 0.02
        .help = "height of the beam (or the irradiated crystal) above the"
                "kapton tape"
        .type = float(allow_none=True)
      sigma = 0.01
        .type = float(allow_none=True)
    }
    rotation_angle_deg {
      value = 1.15
        .help = "angle of the tape from vertical"
        .type = float(allow_none=True)
      sigma = 0.1
        .type = float(allow_none=True)
    }
    kapton_half_width_mm {
      value = 1.5875
        .help = "forward distance from irradiated crystal to edge of tape"
                "nearest detector"
        .type = float(allow_none=True)
      sigma = 0.5
        .type = float(allow_none=True)
    }
    kapton_thickness_mm {
      value = 0.05
        .help = "tape thickness"
        .type = float(allow_none=True)
      sigma = 0.005
        .type = float(allow_none=True)
    }
    smart_sigmas = False
      .help = "apply spot-specific sigma corrections using kapton param sigmas"
      .type = bool
    within_spot_sigmas = True
      .help = "turn this off to get a major speed-up"
      .type = bool
  }
}