Groups | Blog | Home
all groups > sql server data mining > january 2007 >

sql server data mining : Complex query question


Waterboy
1/30/2007 11:15:04 AM
Hello all,

I have a stored proc that grabs a lot of data (4520) rows per day. I need to
pivot the data but the columns are not the same everytime its ran, it depends
on the asset i have selected. i need to covert the data to a csv file, which
im currently doing in the presentation side, but the data is so large that
looping through it is eating ram up like crazy. so i would like to shift the
data using sql

Currently the data comes back like this...
___________________________________________________________
Tag | DataValue | Time | Name | Site
|
-----------------------------------------------------------------------------------
st_act_air | 230.3 | 15:00 | Petyo 123 | Peyto
|
st_act_fuel | 72.5 | 15.00 | Petyo 123 | Peyto
|
------------------------------------------------------------------------------------


I need the data to come back like this:
_________________________________________________________
Site | Name | Time | st_act_air | st_act_fuel
|
---------------------------------------------------------------------------------
Peyto | Peyto 123 | 15:00 | 230.3 | 72.5
|
--------------------------------------------------------------------------------

im using sql server 2000, any one have any ideas or suggestions?
Waterboy
1/31/2007 5:37:01 AM
Thanks for the reply, i will try it out today

Thanks again

[quoted text, click to view]
Dejan Sarka
1/31/2007 9:27:54 AM
Check this dynamic pivoting T-SQL solution:
http://weblogs.sqlteam.com/jeffs/articles/5120.aspx.

--
Dejan Sarka
http://www.solidqualitylearning.com/blogs/

[quoted text, click to view]

Andrey Odegov
2/1/2007 6:48:00 AM
Given your sample data i would solve your problem like this:

SET NOCOUNT ON;
SET ANSI_NULLS ON;
SET QUOTED_IDENTIFIER ON;

-- DECLARE @data TABLE(tag VARCHAR(12) NOT NULL,
-- val DECIMAL(4, 1) NOT NULL CHECK(val > 0),
-- evtime CHAR(5) NOT NULL
-- CHECK(evtime BETWEEN '00:00' AND '23:59'),
-- usr_id VARCHAR(15) NOT NULL,
-- site VARCHAR(10) NOT NULL,
-- PRIMARY KEY(tag, site, usr_id, evtime));

CREATE TABLE #data (tag VARCHAR(12) NOT NULL,
val DECIMAL(4, 1) NOT NULL CHECK(val > 0),
evtime CHAR(5) NOT NULL
CHECK(evtime BETWEEN '00:00' AND '23:59'),
usr_id VARCHAR(15) NOT NULL,
site VARCHAR(10) NOT NULL,
PRIMARY KEY(tag, site, usr_id, evtime));

INSERT INTO #data VALUES('st_act_air', 230.3, '15:00', 'Petyo 123', 'Petyo');
INSERT INTO #data VALUES('st_act_fuel', 22.9, '15:00', 'Petyo 123', 'Petyo');
INSERT INTO #data VALUES('st_act_c00l', 15.8, '15:00', 'Petyo 123', 'Petyo');

INSERT INTO #data VALUES('st_act_air', 302.2, '18:00', 'Petyo 124', 'Petyo');
INSERT INTO #data VALUES('st_act_fuel', 72.5, '18:00', 'Petyo 124', 'Petyo');
INSERT INTO #data VALUES('st_act_bag', 15.1, '18:00', 'Petyo 124', 'Petyo');

DECLARE @table_name SYSNAME;
SET @table_name = '#data';

DECLARE @i_tag_qty INTEGER, @s_tag_qty VARCHAR(10);
DECLARE @tag VARCHAR(12);

DECLARE @sql1 VARCHAR(8000), @sql2 VARCHAR(8000);

DECLARE enum_tags CURSOR READ_ONLY FOR
SELECT DISTINCT tag FROM #data;
OPEN enum_tags;

SET @i_tag_qty = 0;
SET @sql1 = 'SELECT T0.site, T0.usr_id, T0.evtime';
SET @sql2 = ' FROM (SELECT DISTINCT site, usr_id, evtime FROM ' +
@table_name + ') AS T0';

WHILE 1 = 1 BEGIN

FETCH NEXT FROM enum_tags INTO @tag;
IF -1 = @@FETCH_STATUS BREAK;
IF -2 = @@FETCH_STATUS CONTINUE;

SET @i_tag_qty = @i_tag_qty + 1;
SET @s_tag_qty = RTRIM(CAST(@i_tag_qty AS VARCHAR(10)));

SET @sql1 = @sql1 + ', T' + @s_tag_qty + '.val AS ' + @tag;

SET @sql2 = @sql2 + ' LEFT OUTER JOIN ' + @table_name +
' AS T' + @s_tag_qty + ' ON ' +
'T' + @s_tag_qty + '.site = T0.site AND ' +
'T' + @s_tag_qty + '.usr_id = T0.usr_id AND ' +
'T' + @s_tag_qty + '.evtime = T0.evtime AND ' +
'T' + @s_tag_qty + '.tag = ''' + @tag + '''';

END;

CLOSE enum_tags;
DEALLOCATE enum_tags;

--PRINT @sql1;
--PRINT @sql2;

EXEC(@sql1 + @sql2 + ';');

DROP TABLE #data;

---
Andrey Odegov
avodeGOV@yandex.ru
Andrey Odegov
2/3/2007 4:00:01 AM
My first version is incorrect for the following reasons:
1) the CHECK constraint of the evtime column enables to
input non-existent values in the column;
2) the obtained SELECT statement is inefficient. A more
efficient statement is built using the @sql8 and @sql9
variables.

SET NOCOUNT ON;
SET ANSI_NULLS ON;
SET QUOTED_IDENTIFIER ON;

CREATE TABLE #data (tag VARCHAR(12) NOT NULL,
val DECIMAL(4, 1) NOT NULL CHECK(val > 0),
evtime CHAR(5) NOT NULL
CHECK(LEFT(evtime, 2) BETWEEN '00' AND '23' AND
':' = SUBSTRING(evtime, 3, 1) AND
RIGHT(evtime, 2) BETWEEN '00' AND '59'),
usr_id VARCHAR(15) NOT NULL,
site VARCHAR(10) NOT NULL,
PRIMARY KEY(tag, site, usr_id, evtime));

INSERT INTO #data VALUES('st_act_air', 230.3, '15:00', 'Petyo 123', 'Petyo');
INSERT INTO #data VALUES('st_act_fuel', 22.9, '15:00', 'Petyo 123', 'Petyo');
INSERT INTO #data VALUES('st_act_c00l', 15.8, '15:00', 'Petyo 123', 'Petyo');

INSERT INTO #data VALUES('st_act_air', 302.2, '18:00', 'Petyo 124', 'Petyo');
INSERT INTO #data VALUES('st_act_fuel', 72.5, '18:00', 'Petyo 124', 'Petyo');
INSERT INTO #data VALUES('st_act_bag', 15.1, '18:00', 'Petyo 124', 'Petyo');

DECLARE @table_name SYSNAME;
SET @table_name = '#data';

DECLARE @i_tag_qty INTEGER, @s_tag_qty VARCHAR(10);
DECLARE @tag VARCHAR(12);

DECLARE @sql1 VARCHAR(8000), @sql2 VARCHAR(8000);
DECLARE @sql8 VARCHAR(8000), @sql9 VARCHAR(8000);

DECLARE enum_tags CURSOR READ_ONLY FOR
SELECT DISTINCT tag FROM #data;
OPEN enum_tags;

SET @i_tag_qty = 0;

SET @sql1 = 'SELECT T0.site, T0.usr_id, T0.evtime';
SET @sql2 = ' FROM (SELECT DISTINCT site, usr_id, evtime FROM ' +
@table_name + ') AS T0';

SET @sql8 = 'SELECT site, usr_id, evtime';
SET @sql9 = ' FROM ' + @table_name + ' GROUP BY site, usr_id, evtime';

WHILE 1 = 1 BEGIN

FETCH NEXT FROM enum_tags INTO @tag;
IF -1 = @@FETCH_STATUS BREAK;
IF -2 = @@FETCH_STATUS CONTINUE;

SET @i_tag_qty = @i_tag_qty + 1;
SET @s_tag_qty = RTRIM(CAST(@i_tag_qty AS VARCHAR(10)));

SET @sql1 = @sql1 + ', T' + @s_tag_qty + '.val AS ' + @tag;

SET @sql2 = @sql2 + ' LEFT OUTER JOIN ' + @table_name +
' AS T' + @s_tag_qty + ' ON ' +
'T' + @s_tag_qty + '.site = T0.site AND ' +
'T' + @s_tag_qty + '.usr_id = T0.usr_id AND ' +
'T' + @s_tag_qty + '.evtime = T0.evtime AND ' +
'T' + @s_tag_qty + '.tag = ''' + @tag + '''';

SET @sql8 = @sql8 + ', SUM(CASE WHEN tag = ''' + @tag + ''' THEN val ELSE 0
END) AS ' + @tag;

END;

CLOSE enum_tags;
DEALLOCATE enum_tags;

--PRINT @sql1;
--PRINT @sql2;

--PRINT @sql8;
--PRINT @sql9;

--DECLARE @new_line CHAR(2);
--SET @new_line = CHAR(13) + CHAR(10);

EXEC('SET STATISTICS IO ON; ' + @sql1 + @sql2 + '; SET STATISTICS IO OFF;');

EXEC('SET STATISTICS IO ON; ' + @sql8 + @sql9 + '; SET STATISTICS IO OFF;');

DROP TABLE #data;

---
Andrey Odegov
avodeGOV@mail.ru
AddThis Social Bookmark Button