Commit 6d89d38e authored by Mitch Burnett's avatar Mitch Burnett
Browse files

Successful implementation of the PFB into hashpipe and dealer/player.

Showing with 224 additions and 106 deletions
+224 -106
......@@ -9,7 +9,7 @@ else
fi
# Change this to match your destination directory
prefix=~/fresh_hash
prefix=~/hash
# Set library source directories
HASHPIPE_SRC=$FLAG_DIR/lib/hashpipe/src
......
......@@ -26,7 +26,7 @@
#define DEF_NUM_ELEMENTS 64 // System spec for number of elements
#define SAMPLES 4000// Time samples.
#define PFB_OUTPUT_BLOCK_SIZE SAMPLES*PFB_CHANNELS*2
#define PFB_OUTPUT_BLOCK_SIZE SAMPLES*PFB_CHANNELS*DEF_NUM_ELEMENTS*2
// FFT Plan configuration
#define FFTPLAN_RANK 1 // dimension of the transform
......@@ -55,4 +55,4 @@ int doFFT();
int resetDevice(void);
void cleanUp(void);
#endif
\ No newline at end of file
#endif
......@@ -76,11 +76,6 @@ class BeamformerBackend(VegasBackend):
# Read in the additional parameters from the configuration file
self.read_parameters(theBank, theMode)
# Clear shared memory segments
command = "hashpipe_clean_shmem -I %d" %(self.instance)
ps_clean = subprocess.Popen(command.split())
ps_clean.wait()
# Create the ARP table from the mac addresses in dibas.conf under [HPCMACS]
self.name = theMode.backend_name.lower()
self.hpc_macs = hpc_macs
......@@ -93,11 +88,15 @@ class BeamformerBackend(VegasBackend):
# Set some default parameter values
self.requested_weight_file = ''
self.weifile2 = ''
# Add additional dealer-controlled parameters
self.params["int_length"] = self.setIntegrationTime
self.params["weight_file"] = self.setNewWeightFilename
# Set weight flag to default value of 0
self.write_status(WFLAG=str(0))
# Generate ROACH 10-GbE source mac addresses
self.mac_base0 = 0x020200000000 + (2**8)*self.xid + 1
self.mac_base1 = 0x020200000000 + (2**8)*self.xid + 2
......@@ -257,6 +256,15 @@ class BeamformerBackend(VegasBackend):
# Write the beamformer weight filename to shared memory
self.write_status(BWEIFILE=str(self.requested_weight_file))
# Added code: Set flag to update weights in flag_beamform_thread ####
if BWEIFILE is None:
print "Weight file name unchanged."
self.write_status(WFLAG='0')
else:
print "Weight file name changed."
self.write_status(WFLAG='1')
####################################################################
# Print out this information
print now, starttime
......@@ -336,6 +344,16 @@ class BeamformerBackend(VegasBackend):
self.write_status(REQSTI=str(self.requested_integration_time))
self.write_status(BWEIFILE=str(self.requested_weight_file))
self.weifile1 = self.weifile2
self.weifile2 = self.get_status('BWEIFILE')
if self.weifile1 is self.weifile2:
print "Weight file name unchanged."
self.write_status(WFLAG=str(0))
else:
print "Weight file name changed."
self.write_status(WFLAG=str(1))
dt = datetime.utcnow()
dt.replace(second = 0)
dt.replace(microsecond = 0)
......@@ -485,8 +503,8 @@ class BeamformerBackend(VegasBackend):
process_list = process_list + mode_str.split()
# Add threads
thread1 = "-c %d flag_net_thread" % (self.core[0])
thread2 = "-c %d flag_transpose_thread" % (self.core[1])
thread3 = "-c %d flag_correlator_thread" % (self.core[2])
thread2 = "-c %d flag_frb_transpose_thread" % (self.core[1])
thread3 = "-c %d flag_frb_correlator_thread" % (self.core[2])
#thread4 = "-c %d flag_corsave_thread" % (self.core[3])
process_list = process_list + thread1.split()
process_list = process_list + thread2.split()
......@@ -522,7 +540,16 @@ class BeamformerBackend(VegasBackend):
process_list = process_list + thread2.split()
process_list = process_list + thread3.split()
process_list = process_list + thread4.split()
elif self.name == "flag_pfb":
# Add threads
thread1 = "-c %d flag_net_thread" % (self.core[0])
thread2 = "-c %d flag_transpose_thread" % (self.core[1])
thread3 = "-c %d flag_pfb_thread" % (self.core[2])
thread4 = "-c %d flag_pfbsave_thread" % (self.core[3])
process_list = process_list + thread1.split()
process_list = process_list + thread2.split()
process_list = process_list + thread3.split()
process_list = process_list + thread4.split()
print ' '.join(process_list)
self.hpc_process = subprocess.Popen(process_list, stdin=subprocess.PIPE)
......
......@@ -11,33 +11,32 @@
telescope = FLAG
# Which bank is switching master?
who_is_master = BANKB
who_is_master = BANKA
# TCP port for player daemons
dealer_port = 6690
player_port = 6667
[HPCMACS]
10.17.16.208 = 0x7CFE90B92DF0
10.17.16.209 = 0x7CFE90B92BD0
10.17.16.210 = 0x7CFE90B92BD1
10.17.16.211 = 0x7CFE90B92DF1
#10.10.1.5 = 0x7CFE90B1EE50
#10.10.1.6 = 0x7CFE90B1EE51
#10.10.1.7 = 0x7CFE90B1F030
#10.10.1.8 = 0x7CFE90B1F031
#10.10.1.9 = 0x020200000001
#10.10.1.10 = 0x020200000002
#10.10.1.11 = 0x020200000003
#10.10.1.12 = 0x020200000004
#10.10.1.13 = 0x7CFE90B1EEB0
#10.10.1.14 = 0x7CFE90B1EEB1
#10.10.1.15 = 0x7CFE90B1EEE0
#10.10.1.16 = 0x7CFE90B1EEE1
#10.10.1.17 = 0x7CFE90B1EE90
#10.10.1.18 = 0x7CFE90B1EE91
#10.10.1.19 = 0x7CFE90B1EBA0
#10.10.1.20 = 0x7CFE90B1EBA1
10.10.1.1 = 0x7CFE90B1EBC0
10.10.1.2 = 0x7CFE90B1EBC1
10.10.1.3 = 0x7CFE90B1EEC0
10.10.1.4 = 0x7CFE90B1EEC1
10.10.1.5 = 0x7CFE90B1EE50
10.10.1.6 = 0x7CFE90B1EE51
10.10.1.7 = 0x7CFE90B1F030
10.10.1.8 = 0x7CFE90B1F031
10.10.1.9 = 0x020200000001
10.10.1.10 = 0x020200000002
10.10.1.11 = 0x020200000003
10.10.1.12 = 0x020200000004
10.10.1.13 = 0x7CFE90B1EEB0
10.10.1.14 = 0x7CFE90B1EEB1
10.10.1.15 = 0x7CFE90B1EEE0
10.10.1.16 = 0x7CFE90B1EEE1
10.10.1.17 = 0x7CFE90B1EE90
10.10.1.18 = 0x7CFE90B1EE91
10.10.1.19 = 0x7CFE90B1EBA0
10.10.1.20 = 0x7CFE90B1EBA1
# srbs-hpc1-10 = 0x0002C9F558D1
[DEALER]
......@@ -83,20 +82,19 @@ players = BANKA BANKB BANKC BANKD
[BANKA]
# HPC / Player host & port
hpchost = flag3
hpchost = flag4
#hpchost = flag3
player_port = 6677
# ROACH Control:
# Are these even necessary? (RBlack Mar 31, 2016)
#has_roach = true
has_roach = false
katcp_ip = srbsr2-1
katcp_ip = byur2
katcp_port = 7147
# Data flow
data_source_host = srbsr2-1
data_source_host = byur2
data_source_port = 60000
data_destination_host = 10.17.16.208
#data_destination_host = 127.0.0.1
#data_destination_host = 10.18.0.152
data_destination_host = 10.10.1.13
data_destination_port = 60000
# Synthesizer:
synth = none
......@@ -115,40 +113,27 @@ filter_bandwidth_bits = 450, 0x00, 1450, 0x08, 1900, 0x18
# mode from the dealer. (R. Black, 04/05/16)
#
# FLAG-specific parameters
#xid = 12
xid = 0
xid = 12
instance = 0
gpudev = 0
cpus = 0, 1, 2, 3
has_roaches = 1
[BANKB]
# HPC / Player host & port
hpchost = flag3
hpchost = flag4
player_port = 6678
# ROACH Control:
has_roach = false
katcp_ip = srbsr2-1
katcp_port = 7147
# katcp_ip = srbsr2-1
# katcp_port = 7147
# Data flow
# data_source_host = srbsr2-1-10-0
data_source_host = srbsr2-1
data_source_port = 60000
data_destination_host = 10.17.16.211
data_destination_host = 10.10.1.14
data_destination_port = 60000
# Synthesizer:
synth = none
# I'm pretty sure these don't matter if katcp is set as the synthesizer
synth_port = /dev/ttyS1
synth_ref = external
synth_ref_freq = 10000000
synth_vco_range = 2200, 4400
synth_rf_level = 5
synth_options = 0,0,1,0
# synth = katcp
# synth_port = /dev/ttyS1
# synth_ref = external
......@@ -159,45 +144,44 @@ synth_options = 0,0,1,0
# I2C
filter_bandwidth_bits = 450, 0x00, 1450, 0x08, 1900, 0x18
# FLAG-specific parameters
xid = 3
xid = 13
instance = 1
gpudev = 0
cpus = 4, 1, 5, 3
has_roaches = 0
[BANKC]
# HPC / Player host & port
hpchost = flag3
hpchost = flag4
player_port = 6679
# ROACH Control:
has_roach = false
# katcp_ip = srbsr2-1
# katcp_port = 7147
katcp_ip = byur2
katcp_port = 7147
# Data flow
# data_source_host = srbsr2-1-10-0
data_source_host = byur2
data_source_port = 60000
data_destination_host = 10.17.16.209
data_destination_host = 10.10.1.15
data_destination_port = 60000
# Synthesizer:
# synth = katcp
# synth_port = /dev/ttyS1
# synth_ref = external
# synth_ref_freq = 10000000
# synth_vco_range = 2200, 4400
# synth_rf_level = 5
# synth_options = 0,0,1,0
synth = none
# I'm pretty sure these don't matter if katcp is set as the synthesizer
synth_port = /dev/ttyS1
synth_ref = external
synth_ref_freq = 10000000
synth_vco_range = 2200, 4400
synth_rf_level = 5
synth_options = 0,0,1,0
# I2C
filter_bandwidth_bits = 450, 0x00, 1450, 0x08, 1900, 0x18
# FLAG-specific parameters
xid = 1
xid = 14
instance = 2
gpudev = 1
cpus = 6, 7, 8, 9
has_roaches = 0
[BANKD]
# HPC / Player host & port
hpchost = flag3
hpchost = flag4
player_port = 6680
# ROACH Control:
has_roach = false
......@@ -206,7 +190,7 @@ has_roach = false
# Data flow
# data_source_host = srbsr2-1-10-0
data_source_port = 60000
data_destination_host = 10.17.16.210
data_destination_host = 10.10.1.16
data_destination_port = 60000
# Synthesizer:
# synth = katcp
......@@ -219,14 +203,15 @@ data_destination_port = 60000
# I2C
filter_bandwidth_bits = 450, 0x00, 1450, 0x08, 1900, 0x18
# FLAG-specific parameters
xid = 2
xid = 15
instance = 3
gpudev = 1
cpus = 10, 7, 11, 9
has_roaches = 0
################################### FLAG CORR MODE #############################################
################################### FLAG HICORR MODE #############################################
[FLAG_HICORR_MODE]
# These values get loaded directly into status shared memory
......@@ -236,7 +221,8 @@ BACKEND = FLAG
MODENAME = hi_correlator
hpc_program=hashpipe
hpc_program_flags=-p flag_gpu
hpc_program_flags=-p flag_x
#fits_process = dummy_fits_writer
fits_process = bfFitsWriter
# IP and MAC Addresses
......@@ -269,8 +255,9 @@ BACKEND = FLAG
MODENAME = cal_correlator
hpc_program=hashpipe
hpc_program_flags=-p flag_gpu
fits_process = bfFitsWriter
hpc_program_flags=-p flag_x
fits_process = dummy_fits_writer
#fits_process = bfFitsWriter
# IP and MAC Addresses
fabric_port = 60000
......@@ -284,15 +271,44 @@ filter_bw = 1450
frequency = 1500
nchan = 1024
hpc_fifo_name = /tmp/wouldntyouliketoknow.fifo
needed_arm_delay = 1
needed_arm_delay = 2
gigabit_interface_name = 10.2.118.123
dest_ip_register_name = bart
dest_port_register_name = lisa
master_slave_sel = 0,0,0,0,0,0
sim3 = 0
################################### FLAG PFB MODE #############################################
[FLAG_PFB_MODE]
# These values get loaded directly into status shared memory
shmkeys = BACKEND,MODENAME
BACKEND = FLAG
MODENAME = flag_pfb
hpc_program=hashpipe
hpc_program_flags=-p flag_f
fits_process = dummy_fits_writer
#fits_process = bfFitsWriter
# IP and MAC Addresses
fabric_port = 60000
bof_file = flag_sim3_v1.bof
arm_phase = sync_gen_msync_in,0x0,sync_gen_msync_in,0x1,sync_gen_msync_in,0x0
# These are currently mandatory values
hwexposr = 0.000500395
filter_bw = 1450
frequency = 1500
nchan = 1024
hpc_fifo_name = /tmp/wouldntyouliketoknow.fifo
needed_arm_delay = 2
gigabit_interface_name = 10.2.118.123
dest_ip_register_name = bart
dest_port_register_name = lisa
master_slave_sel = 0,0,0,0,0,0
################################### FLAG FRBCORR MODE #############################################
[FLAG_FRBCORR_MODE]
......@@ -304,8 +320,9 @@ BACKEND = FLAG
MODENAME = frb_correlator
hpc_program=hashpipe
hpc_program_flags=-p flag_gpu
fits_process = bfFitsWriter
hpc_program_flags=-p flag_x_frb
fits_process = dummy_fits_writer
# fits_process = bfFitsWriter
# IP and MAC Addresses
fabric_port = 60000
......@@ -337,7 +354,8 @@ BACKEND = FLAG
MODENAME = pulsar_beamformer
hpc_program=hashpipe
hpc_program_flags=-p flag_beamformer
hpc_program_flags=-p flag_b
# fits_process = dummy_fits_writer
fits_process = bfFitsWriter
# IP and MAC Addresses
......@@ -390,3 +408,38 @@ gigabit_interface_name = 10.2.118.123
dest_ip_register_name = bart
dest_port_register_name = lisa
master_slave_sel = 0,0,0,0,0,0
################################### FLAG DUAL MODE #############################################
[FLAG_DUAL_MODE]
# These values get loaded directly into status shared memory
shmkeys = BACKEND,MODENAME
BACKEND = FLAG
MODENAME = flag_dual_mode
hpc_program=hashpipe
hpc_program_flags=-p flag_bx
fits_process = dummy_fits_writer
#fits_process = bfFitsWriter
# IP and MAC Addresses
fabric_port = 60000
bof_file = flag_sim3_v1.bof
arm_phase = sync_gen_msync_in,0x0,sync_gen_msync_in,0x1,sync_gen_msync_in,0x0
# These are currently mandatory values
hwexposr = 0.000500395
filter_bw = 1450
frequency = 1500
nchan = 1024
hpc_fifo_name = /tmp/wouldntyouliketoknow.fifo
needed_arm_delay = 2
gigabit_interface_name = 10.2.118.123
dest_ip_register_name = bart
dest_port_register_name = lisa
master_slave_sel = 0,0,0,0,0,0
......@@ -61,14 +61,14 @@ flag_x_frb_threads = flag_net_thread.c \
flag_frb_correlator_thread.c \
flag_frb_corsave_thread.c
flag_fx_threads = flag_net_thread.c \
flag_transpose_thread.c \
flag_pfb_thread.c \
flag_pfb_correlator_thread.c \
flag_pfb_corsave_thread.c
#flag_fx_threads = flag_net_thread.c \
# flag_transpose_thread.c \
# flag_pfb_thread.c \
# flag_pfb_correlator_thread.c \
# flag_pfb_corsave_thread.c
# This lists all of the plugins that will be created
lib_LTLIBRARIES = flag_x.la flag_b.la flag_f.la flag_x_frb.la flag_bx.la flag_fx.la
lib_LTLIBRARIES = flag_x.la flag_b.la flag_f.la flag_x_frb.la flag_bx.la
# flag_x.la sources and libraries
flag_x_la_SOURCES = $(flag_databuf) $(flag_x_threads) $(fifo_codes)
......@@ -109,12 +109,12 @@ flag_x_frb_la_LDFLAGS += -L"@XGPU_FRB_LIBDIR@" -Wl,-rpath,"@XGPU_FRB_LIBDIR@"
flag_x_frb_la_LDFLAGS += -L"@HASHPIPE_LIBDIR@" -Wl,-rpath,"@HASHPIPE_LIBDIR@"
# flag_fx.la sources and libraries
flag_fx_la_SOURCES = $(flag_databuf) $(flag_fx_threads) $(fifo_codes)
flag_fx_la_LIBADD = -lrt -lxgpupfb -lcublas -L/usr/local/cuda/lib64 -lcfitsio -lflagpfb -lcufft -lcudart
flag_fx_la_LDFLAGS = -avoid-version -module -shared -export-dynamic --enable-shared
flag_fx_la_LDFLAGS += -L"@XGPU_PFB_LIBDIR@" -Wl,-rpath,"@XGPU_PFB_LIBDIR@"
flag_fx_la_LDFLAGS += -L"@HASHPIPE_LIBDIR@" -Wl,-rpath,"@HASHPIPE_LIBDIR@"
flag_fx_la_LDFLAGS += -L"@FLAGPFB_LIBDIR@" -Wl, -rpath,"@FLAGPFB_LIBDIR@"
#flag_fx_la_SOURCES = $(flag_databuf) $(flag_fx_threads) $(fifo_codes)
#flag_fx_la_LIBADD = -lrt -lxgpupfb -lcublas -L/usr/local/cuda/lib64 -lcfitsio -lflagpfb -lcufft -lcudart
#flag_fx_la_LDFLAGS = -avoid-version -module -shared -export-dynamic --enable-shared
#flag_fx_la_LDFLAGS += -L"@XGPU_PFB_LIBDIR@" -Wl,-rpath,"@XGPU_PFB_LIBDIR@"
#flag_fx_la_LDFLAGS += -L"@HASHPIPE_LIBDIR@" -Wl,-rpath,"@HASHPIPE_LIBDIR@"
#flag_fx_la_LDFLAGS += -L"@FLAGPFB_LIBDIR@" -Wl, -rpath,"@FLAGPFB_LIBDIR@"
# Installed scripts
......
......@@ -330,7 +330,7 @@ static inline int64_t process_packet(flag_input_databuf_t * db, struct hashpipe_
*/
// print_pkt_header(&pkt_header);
//print_pkt_header(&pkt_header);
......
......@@ -121,15 +121,20 @@ static void * run(hashpipe_thread_args_t * args) {
// Get block's starting mcnt for output block
db_out->block[curblock_out].header.mcnt = start_mcnt;
db_out->block[curblock_out].header.good_data = good_data;
printf("PFB: Wrote header info...\n");
// Mark output block as full and advance
flag_gpu_pfb_output_databuf_set_filled(db_out, curblock_out);
printf("PFB: Marked block %d as filled...\n", curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
start_mcnt = last_mcnt + 1;
last_mcnt = start_mcnt + Nm - 1;
printf("PFB: Attempting to mark block %d as free...\n", curblock_in);
// Mark input block as free
flag_gpu_input_databuf_set_free(db_in, curblock_in);
printf("PFB: Marked block %d as free...\n", curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
}
else if (cur_state == CLEANUP) {
......
......@@ -61,16 +61,23 @@ static void * run(hashpipe_thread_args_t * args) {
break;
}
}
printf("Getting ready to save...\n");
uint64_t start_mcnt = db_in->block[curblock_in].header.mcnt;
int good_data_flag = (int)(db_in->block[curblock_in].header.good_data);
float * pfb_out_data = (float *)db_in->block[curblock_in].data;
printf("Creating filename...\n");
char filename[256];
sprintf(filename, "%s/pfb_%d_mcnt_%lld.out", data_dir, instance_id, (long long)start_mcnt);
FILE * filePtr = fopen(filename, "w");
printf("Saving output file to: %s\n", filename);
FILE * filePtr = fopen(filename, "wb");
if(filePtr == NULL) {
printf("DUM DUM...\n");
}
printf("Writing good data...\n");
fwrite(&good_data_flag, sizeof(int), 1, filePtr);
printf("Writing data out...\n");
fwrite(pfb_out_data, sizeof(float), PFB_OUTPUT_BLOCK_SIZE, filePtr);
fclose(filePtr);
......
......@@ -34,8 +34,9 @@ sigma2 = kb*Tsys*BW; % Noise power per channel
% 3 -> Send all ones
% 4 -> Send chirp
% 5 -> Send spatially correlated data in a single bin
% 6 -> Send complex sinusodial data
% else -> Send all zeros
data_flag = 5;
data_flag = 6;
% Sinusoid parameters (only used if data_flag = 2)
% It should be noted that the phase of the sinusoid will not change between
......@@ -63,7 +64,6 @@ c_time_per_bin = c_ntime/c_num_bins;
% Correlated data parameters (only used if data_flag = 5)
% kw = Karl Warnick
kw_bin = 1;
kw_xid = floor((kw_bin - 1)/Nbin_per_x) + 1;
kw_bin_r = mod(kw_bin - 1, Nbin_per_x) + 1;
......@@ -79,7 +79,8 @@ end
% create time samples
rng(1);
kwNs = 4000*40;
N = 4000;
kwNs = 4000;
kw_z = (randn(Nel, kwNs) + 1j*randn(Nel,kwNs))/sqrt(2);
kwM = sqrtm(kwR);
kw_x = kwM*kw_z;
......@@ -96,10 +97,25 @@ kw_Rhat = (kw_x_quant*kw_x_quant')/kwNs;
save('matlab_corr.mat', 'kw_Rhat');
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Case 6 - Complex Sinusoid
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
cs_Ns = 4000;
cs_freq = 0.25;
cs_n = 0:cs_Ns-1;
cs_re = 127 * (0.1 * cos(2*pi*cs_freq*cs_n));
cs_im = 127 * (0.1 * sin(2*pi*cs_freq*cs_n));
% Create UDP sockets - 1 IP address per Xengine (xid)
for xid = 1:Nxengines
remoteHost = ['10.17.16.', num2str(207+xid)];
remoteHost = ['10.10.1.', num2str(xid)];
%if xid == 1
% remoteHost = '10.10.1.14';
%end
%if xid == 14
% remoteHost = '10.10.1.1';
%end
sock(xid) = udp(remoteHost, 'RemotePort', 60000, 'LocalPort', 60001);
set(sock(xid), 'OutputBufferSize', 9000);
set(sock(xid), 'OutputDatagramPacketSize', 9000);
......@@ -109,9 +125,9 @@ end
% Generate packet payloads
mcnt = 0; % Each mcnt represents 20 packets across all F-engines in the
% same time frame
for mcnt = [0:7000] %while mcnt <= 10000
for mcnt = [0:200, 401] %while mcnt <= 10000
disp(['Sending mcnt = ', num2str(mcnt)]);
for xid = 1:1 % Set to a single X-engine for single HPC testing (Richard B.)
for xid = 14:14 % Set to a single X-engine for single HPC testing (Richard B.)
for fid = 1:Nfengines
w_idx = 1;
......@@ -194,6 +210,16 @@ for mcnt = [0:7000] %while mcnt <= 10000
data(:, 2, kw_bin_r, :) = kw_x_imag(f_idxs,t_idxs);
data(data < 0) = 2^8 + data(data < 0);
end
case 6 % Send complex sinusoidal data
t_idxs = mod(mcnt*20 + 1:(mcnt+1)*20, cs_Ns);
t_idxs(t_idxs == 0) = kwNs;
for cs_bin = 1:Nbin_per_x
for cs_ele = 1:Nin_per_f
data(cs_ele,1,cs_bin,:) = cs_re(t_idxs);
data(cs_ele,2,cs_bin,:) = cs_im(t_idxs);
end
end
data(data < 0) = 2^8 + data(data < 0);
otherwise % Send all zeros
end
data = uint8(data);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment