#################################################
#######       Python setup                #######
#################################################
# Install Python 2.7 with the following libraries (required version in parentheses):
#  - numpy (>=1.12.1)
#  - scipy (>=0.19.0)
#  - pandas (>=0.20.1)
#  - matplotlib (>=2.0.2)
#  - scikit-learn (>=0.18.1)
#  - biopython (>=1.68)
#  - fastcluster (>=1.1.20)
#  - pyyaml (>= 3.11)
#  - networkx (>=1.11)
#
# Conda (https://conda.io/docs/user-guide/install/index.html) provides a way to
# quickly install python and the required libraries. If you install conda, then
# the required packages above can be obtained from the Anaconda Cloud
# (https://anaconda.org/).


#################################################
#######       BEAN-counter setup          #######
#################################################

# Download the latest version of BEAN-counter from here: https://github.com/csbio/BEAN-counter/releases

# First, cd into the BEAN-counter directory
# cd BEAN-counter-x.x.x
# Download the gene barcode and amplicon structure files for the sample dataset
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/gene_barcode_files/Sc_hap_del_mp_hs_v1.txt -P data/gene_barcode_files
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/amplicon_struct_files/Sc_hap_MP-WG-HET-MoBY.yaml -P data/amplicon_struct_files


#################################################
#######       Dataset processing setup    #######
#################################################

# Create and cd into your working directory
# mkdir -p <dir>
# cd <dir>

# Make sure your $BARSEQ_PATH environmental variable is set to the location
# of the BEAN-counter software, and that $BARSEQ_PATH/master_scripts is
# in your $PATH

# Run the interactive screen setup script
setup_screen.py -i

# Download the sample table
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/sample_table/sample_table.txt -P sample_table

# Download the raw sequencing data to the appropriate lane folders
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane01/lane01_R1.fastq.gz -P raw/lane1
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane02/lane02_R1.fastq.gz -P raw/lane2
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane03/lane03_R1.fastq.gz -P raw/lane3
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane04/lane04_R1.fastq.gz -P raw/lane4
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane05/lane05_R1.fastq.gz -P raw/lane5
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane06/lane06_R1.fastq.gz -P raw/lane6
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane07/lane07_R1.fastq.gz -P raw/lane7
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane08/lane08_R1.fastq.gz -P raw/lane8
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/raw/lane09/lane09_R1.fastq.gz -P raw/lane9


#################################################
#######       Interaction scoring         #######
#################################################

# Tip: send all script output to a log file for permanent record-keeping
# example: process_screen.py config_files/config_file.yaml &>> logs/logfile.txt
mkdir logs

# Initial round of interaction scoring
process_screen.py config_files/config_file.yaml 
visualize_zscore_matrices.py config_files/config_file.yaml Gene_name,Strain_ID \
	screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 

# Download filtered gene barcode and sample tables
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/barcodes/barcodes_filtered.txt -P barcodes
wget http://lovelace-umh.cs.umn.edu/BEAN-counter/example_dataset/sample_table/sample_table_filtered.txt -P sample_table
# Also, modify config_files/config_file.yaml so that it points to barcodes/barcodes_filtered.txt and sample_table/sample_table_filtered.txt

# Perform final filtering and scoring steps again, on filtered dataset
process_screen.py --start 5 config_files/config_file.yaml 

# Generate and inspect clustered heat map
visualize_zscore_matrices.py config_files/config_file.yaml Gene_name,Strain_ID \
	screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 

# Create a heat map of just the control compounds
reduce_dataset.py --column "control?" \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev.dump.gz \
	sample_table/sample_table_filtered.txt \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev_DMSO.dump.gz 

visualize_dataset.py output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev_DMSO.dump.gz \
	DMSO_controls \
	barcodes/barcodes.txt \
	sample_table/sample_table.txt \
	Gene_name,Strain_ID \
	screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 

# Final round of interaction scoring, starting immediately after raw data processing
process_screen.py --start 2 config_files/config_file.yaml 

# Generate and inspect clustered heat map
visualize_zscore_matrices.py config_files/config_file.yaml Gene_name,Strain_ID \
	screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 

#################################################
#######       Post-processing             #######
#################################################

# Check replicate correlations
check_batch_effects.py -incl not_pos_neg_control \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev.dump.gz \
	sample_table/sample_table_filtered.txt name none 

# Check index tag correlations
check_batch_effects.py -incl not_pos_control \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev.dump.gz \
	sample_table/sample_table_filtered.txt index_tag name 

# Check lane correlations
check_batch_effects.py -incl not_pos_control \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev.dump.gz \
	sample_table/sample_table_filtered.txt lane name 

# Perform SVD correction
svd_correction.py -incl not_pos_control \
	output/interactions/all_lanes_filtered/all_lanes_filtered_scaled_dev.dump.gz \
	sample_table/sample_table_filtered.txt 4 output/svd_correction 

# Generate clustered heat maps for different numbers of SVD components removed
visualize_stacked_zscore_matrices.py output/svd_correction/svd_corrected_datasets.dump.gz \
	svd_correction barcodes/barcodes_filtered.txt sample_table/sample_table_filtered.txt \
	Gene_name,Strain_ID screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 

# Check replicate correlations for different numbers of SVD components removed
check_batch_effects.py -incl not_pos_neg_control \
	output/svd_correction/svd_corrected_datasets.dump.gz \
	sample_table/sample_table_filtered.txt name none 

# Extract the dataset with 1 SVD component removed
extract_dataset.py output/svd_correction/svd_corrected_datasets.dump.gz 1 
# Generate a clustered heat map
visualize_dataset.py output/svd_correction/svd_corrected_datasets/1_components_removed.dump.gz \
	final_dataset barcodes/barcodes_filtered.txt sample_table/sample_table_filtered.txt \
	Gene_name,Strain_ID screen_name,expt_id,name,lane,index_tag_plate,index_tag_well,index_tag 
# Export to text
dataset_to_text.py output/svd_correction/svd_corrected_datasets/1_components_removed.dump.gz 


