Version 1.1:

- BAM output support,
- standart output support,
- RG tag added,
- lots of bugs fixed.

Author: agudys

src/common/defs.h

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -34,7 +34,7 @@
 #include <string>
 using namespace std;
 
-#define MAPPER_VERSION		"1.0 (2017-12-24)"
+#define MAPPER_VERSION		"1.1 (2018-07-10)"
 #define MAPPER_ID			"Whisper"
 #define MAPPER_NAME			"Whisper"
 

src/common/fastq_reader.cpp

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -656,7 +656,7 @@ void CPrePostBaseFastqReader::load_single_file(file_name_no_t file_name_no)
 
 	mem_pool->Reserve(part);
 	if(verbosity_level >= 3)
-		cout << "FastqReader: " << file_name_no.file_name1 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+		cerr << "FastqReader: " << file_name_no.file_name1 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 	uint32_t no_reads;
 
@@ -668,7 +668,7 @@ void CPrePostBaseFastqReader::load_single_file(file_name_no_t file_name_no)
 
 		mem_pool->Reserve(part);
 		if(verbosity_level >= 3)
-			cout << "FastqReader: " << file_name_no.file_name1 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+			cerr << "FastqReader: " << file_name_no.file_name1 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 		++part_no;
 	}
@@ -696,7 +696,7 @@ void CPrePostBaseFastqReader::load_pair_files_thr(file_name_no_t file_name_no)
 	uint32_t no_reads1, no_reads2;
 
 	if(verbosity_level >= 3)
-		cout << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+		cerr << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 	while (true)
 	{
@@ -716,7 +716,7 @@ void CPrePostBaseFastqReader::load_pair_files_thr(file_name_no_t file_name_no)
 
 		if ((p1_status ^ p2_status) || (no_reads1 != no_reads2 && no_reads == 0))
 		{
-			cout << "Error: Different number of reads in paired files!\n";
+			cerr << "Error: Different number of reads in paired files!\n";
 			exit(1);
 		}
 
@@ -730,7 +730,7 @@ void CPrePostBaseFastqReader::load_pair_files_thr(file_name_no_t file_name_no)
 		process_pair_blocks(file_name_no, part_no, id1, part1, part_filled1, id2, part2, part_filled2);
 
 		if (verbosity_level >= 3)
-			cout << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+			cerr << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 		++part_no;
 	}
@@ -754,7 +754,7 @@ void CPrePostBaseFastqReader::load_pair_files(file_name_no_t file_name_no)
 	mem_pool->Reserve(part1, part2);
 
 	if(verbosity_level >= 3)
-		cout << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+		cerr << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 	while(fqr1->GetPartInfo(part1, part_filled1, no_reads1) && fqr2->GetPartInfo(part2, part_filled2, no_reads2))
 	{
@@ -766,7 +766,7 @@ void CPrePostBaseFastqReader::load_pair_files(file_name_no_t file_name_no)
 		mem_pool->Reserve(part1, part2);
 
 		if(verbosity_level >= 3)
-			cout << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
+			cerr << "FastqReader: " << file_name_no.file_name1 << " : " << file_name_no.file_name2 << "  (free blocks: " << mem_pool->GetAvailableParts() << ")\n";
 
 		++part_no;
 	}
@@ -878,7 +878,7 @@ bool CPostFastqReader::is_new_group(read_id_t prev_id, read_id_t new_id)
 void CPostFastqReader::end_of_thread()
 {
 	if(verbosity_level >= 2)
-		cout << "PostFastqReader end_of_thread\n";
+		cerr << "PostFastqReader end_of_thread\n";
 	q_res_ids->MarkCompleted();
 	joiner_mgr->MarkFastqReaderCompleted();
 }

src/common/fastq_reader.h

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 

src/common/idstore.cpp

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -119,7 +119,7 @@ read_id_t CIDStore::RegisterBlock(uint32_t file_no, string file_name, uint32_t f
 	}
 
 	if(verbosity_level > 2)
-		cout << "IDStore:-register " << hex << id << "\n";
+		cerr << "IDStore:-register " << hex << id << "\n";
 
 	return id;
 }
@@ -140,7 +140,7 @@ read_id_t CIDStore::GetBlockID(uint32_t file_no, string file_name, uint32_t file
 	id += ((read_id_t) file_part & sub_block_mask) << in_block_bits;
 
 	if(verbosity_level > 2)
-		cout << "IDStore-get: " << hex << id << "\n";
+		cerr << "IDStore-get: " << hex << id << "\n";
 
 	return id;
 }

src/common/idstore.h

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 

src/common/joiner_mgr.cpp

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -114,7 +114,7 @@ void CJoinerMgr::MarkFastqReaderCompleted()
 void CJoinerMgr::send_bin_for_processing(uint32_t bin_id)
 {
 	if(verbosity_level >= 3)
-		cout << "JoinerMgr: sending result group for converting into SAM: " << bin_id << " size of data: " << data.size() << "\n";
+		cerr << "JoinerMgr: sending result group for converting into SAM: " << bin_id << " size of data: " << data.size() << "\n";
 
 	q_map_reads->Push(data[bin_id]);
 
@@ -129,7 +129,7 @@ void CJoinerMgr::send_bin_for_processing(uint32_t bin_id)
 void CJoinerMgr::complete()
 {
 	if(verbosity_level >= 3)
-		cout << "JoinerMgr: mapping results reading finished\n";
+		cerr << "JoinerMgr: mapping results reading finished\n";
 
 	q_map_reads->MarkCompleted();
 }

src/common/joiner_mgr.h

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 

src/common/mmgr.h

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -18,11 +18,128 @@
 #include <condition_variable>
 #include <mutex>
 #include <iostream>
+#include <algorithm>
 
 //#define DISABLE_MEMORY_MONITOR
 
 using namespace std;
 
+// ************************************************************************************
+// CPtrPool
+// ************************************************************************************
+class CPtrPool
+{
+	typedef struct {
+		uchar_t *ptr;
+		uint64_t size;
+		bool reserved;
+	} pool_item_t;
+	
+	pool_item_t *pool;
+	int pool_size;
+	int no_reserved;
+
+	int64_t alloc_mem;
+
+	mutable mutex mtx;							// The mutex to synchronise on
+	condition_variable cv;						// The condition to wait for
+
+public:
+	CPtrPool(int _pool_size)
+	{
+		alloc_mem = 0;
+
+		pool_size = _pool_size;
+		pool = new pool_item_t[pool_size];
+		no_reserved = 0;
+
+		for (int i = 0; i < pool_size; ++i)
+		{
+			pool[i].size = 1;
+			pool[i].ptr = new uchar_t[pool[i].size];
+			pool[i].reserved = false;
+
+			alloc_mem += 1;
+		}
+	}
+
+	~CPtrPool()
+	{
+		for (int i = 0; i < pool_size; ++i)
+		{
+			delete[] pool[i].ptr;
+
+			alloc_mem -= pool[i].size;
+//			cerr << "Ptr_pool destructor: " + to_string(alloc_mem >> 20) + "MB\n";
+		}
+
+		delete[] pool;
+	}
+
+	uchar_t* Allocate(uint64_t requested_size)
+	{
+		unique_lock<mutex> lck(mtx);
+		cv.wait(lck, [this] {return no_reserved < pool_size; });
+
+		// Try to find matching block
+		for (int i = 0; i < pool_size; ++i)
+			if (!pool[i].reserved && pool[i].size >= requested_size)
+			{
+				pool[i].reserved = true;
+				++no_reserved;
+				return pool[i].ptr;
+			}
+
+		for (int i = pool_size - 1; i >= 0; --i)
+			if (!pool[i].reserved)
+			{
+				delete[] pool[i].ptr;
+
+				alloc_mem -= pool[i].size;
+//				cerr << "Ptr_pool alloc- : " + to_string(alloc_mem >> 20) + "MB\n";
+
+				// 10% overhead to avoid too many reallocations
+				pool[i].size = (uint64_t)(requested_size * 1.1);
+
+				pool[i].ptr = new uchar_t[pool[i].size];
+				pool[i].reserved = true;
+				++no_reserved;
+				alloc_mem += pool[i].size;
+//				cerr << "Ptr_pool alloc+ : " + to_string(alloc_mem >> 20) + "MB\n";
+
+				auto p = pool[i].ptr;
+
+				sort(pool, pool + pool_size, [](pool_item_t &x, pool_item_t &y) {return x.size < y.size; });
+
+				return p;
+			}
+
+//		cerr << "Cannot allocate in ptr pool: " + to_string(no_reserved) + " : " + to_string(pool_size) + "\n";
+//		exit(1);
+
+		return nullptr;
+	}
+
+	void Release(uchar_t *p)
+	{
+		lock_guard<mutex> lck(mtx);
+		
+		for (int i = 0; i < pool_size; ++i)
+			if (pool[i].ptr == p)
+			{
+				pool[i].reserved = false;
+
+				if (no_reserved-- == pool_size)
+					cv.notify_one();
+				return;
+			}
+	
+//		cerr << "Cannot find pointer\n";
+//		exit(1);
+	}
+};
+
+
 // ************************************************************************************
 // CMemoryPool
 // ************************************************************************************
@@ -112,7 +229,7 @@ template<typename T> class CMemoryPool
 		unique_lock<mutex> lck(mtx);
 		cv.wait(lck, [this]{return n_parts_free > 0;});
 
-		part = buffer + stack[--n_parts_free]*part_size;
+		part = reinterpret_cast<T*>(buffer + stack[--n_parts_free]*part_size);
 	}
 
 	// ************************************************************************************
@@ -121,8 +238,8 @@ template<typename T> class CMemoryPool
 		unique_lock<mutex> lck(mtx);
 		cv.wait(lck, [this]{return n_parts_free > 1;});
 
-		part1 = buffer + stack[--n_parts_free]*part_size;
-		part2 = buffer + stack[--n_parts_free]*part_size;
+		part1 = reinterpret_cast<T*>(buffer + stack[--n_parts_free]*part_size);
+		part2 = reinterpret_cast<T*>(buffer + stack[--n_parts_free]*part_size);
 	}
 
 	// ************************************************************************************
@@ -171,7 +288,7 @@ class CMemoryMonitor
 
 	void show_status(int64_t n)
 	{
-		cout << "Memory monitor: " << max_memory << "  " << memory_in_use << "  : " << n << "\n";
+		cerr << "Memory monitor: " << max_memory << "  " << memory_in_use << "  : " << n << "\n";
 	}
 
 public:

src/common/params.cpp

@@ -4,8 +4,8 @@
 // 
 // Authors: Sebastian Deorowicz, Agnieszka Debudaj-Grabysz, Adam Gudys
 // 
-// Version : 1.0
-// Date    : 2017-12-24
+// Version : 1.1
+// Date    : 2018-07-10
 // License : GNU GPL 3
 // *******************************************************************************************
 
@@ -55,7 +55,7 @@ ostream& operator<<(ostream &out, const CParams &p)
 	out << "max_bin_part_size             : " << p.max_bin_part_size << endl;
 	out << "max_cigar_len                 : " << p.max_cigar_len << endl;
 	out << "max_fastq_rec_length          : " << p.max_fastq_rec_length << endl;
-	out << "max_frac_errors               : " << p.max_frac_errors << endl;
+	out << "max_no_errors                 : " << p.max_no_errors << endl;
 	out << "max_group_part_size           : " << p.max_group_part_size << endl;
 	out << "max_mapping_memory            : " << p.max_mapping_memory << endl;
 	out << "max_mate_distance             : " << p.max_mate_distance << endl;
@@ -85,6 +85,7 @@ ostream& operator<<(ostream &out, const CParams &p)
 	out << "project_name                  : " << p.project_name << endl;
 	out << "read_len                      : " << p.read_len << endl;
 	out << "res_group_size                : " << p.res_group_size << endl;
+	out << "read_group_line               : " << p.read_group_line << endl;
 	out << "sa_in_ram                     : " << p.sa_in_ram << endl;
 	out << "sa_prefix_overhead            : " << p.sa_prefix_overhead << endl;
 	out << "sam_buffer_memory             : " << p.sam_buffer_memory << endl;